Annotation of /trunk/AssetServer/Extensions/AmazonS3/AmazonS3Storage.cs
Parent Directory
| 
Revision Log
Revision 69 - (view) (download)
| 1 : | jhurliman | 49 | /* |
| 2 : | * Copyright (c) 2008 Intel Corporation | ||
| 3 : | * All rights reserved. | ||
| 4 : | * Redistribution and use in source and binary forms, with or without | ||
| 5 : | * modification, are permitted provided that the following conditions | ||
| 6 : | * are met: | ||
| 7 : | * | ||
| 8 : | * -- Redistributions of source code must retain the above copyright | ||
| 9 : | * notice, this list of conditions and the following disclaimer. | ||
| 10 : | * -- Redistributions in binary form must reproduce the above copyright | ||
| 11 : | * notice, this list of conditions and the following disclaimer in the | ||
| 12 : | * documentation and/or other materials provided with the distribution. | ||
| 13 : | * -- Neither the name of the Intel Corporation nor the names of its | ||
| 14 : | * contributors may be used to endorse or promote products derived from | ||
| 15 : | * this software without specific prior written permission. | ||
| 16 : | * | ||
| 17 : | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | ||
| 18 : | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | ||
| 19 : | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A | ||
| 20 : | * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE INTEL OR ITS | ||
| 21 : | * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, | ||
| 22 : | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, | ||
| 23 : | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR | ||
| 24 : | * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF | ||
| 25 : | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING | ||
| 26 : | * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS | ||
| 27 : | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
| 28 : | */ | ||
| 29 : | |||
| 30 : | using System; | ||
| 31 : | using System.Collections.Generic; | ||
| 32 : | using System.Net; | ||
| 33 : | using System.IO; | ||
| 34 : | using ExtensionLoader; | ||
| 35 : | using ExtensionLoader.Config; | ||
| 36 : | using OpenMetaverse; | ||
| 37 : | using OpenMetaverse.StructuredData; | ||
| 38 : | using LitS3; | ||
| 39 : | |||
| 40 : | namespace AssetServer.AmazonS3Storage | ||
| 41 : | { | ||
| 42 : | public class AmazonS3Storage : IExtension<AssetServer>, IStorageProvider | ||
| 43 : | { | ||
| 44 : | const string EXTENSION_NAME = "AmazonS3Storage"; // Used in metrics reporting | ||
| 45 : | |||
| 46 : | AssetServer server; | ||
| 47 : | S3Service s3; | ||
| 48 : | string bucketName; | ||
| 49 : | string cloudFrontHost; | ||
| 50 : | |||
| 51 : | public AmazonS3Storage() | ||
| 52 : | { | ||
| 53 : | } | ||
| 54 : | |||
| 55 : | public void Start(AssetServer server) | ||
| 56 : | { | ||
| 57 : | this.server = server; | ||
| 58 : | bool useCloudFront = false; | ||
| 59 : | |||
| 60 : | try | ||
| 61 : | { | ||
| 62 : | IConfig amazonConfig = server.ConfigFile.Configs["Amazon"]; | ||
| 63 : | |||
| 64 : | s3 = new S3Service(); | ||
| 65 : | s3.AccessKeyID = amazonConfig.GetString("AccessKeyID"); | ||
| 66 : | s3.SecretAccessKey = amazonConfig.GetString("SecretAccessKey"); | ||
| 67 : | bucketName = amazonConfig.GetString("BucketName"); | ||
| 68 : | useCloudFront = amazonConfig.GetBoolean("UseCloudFront"); | ||
| 69 : | } | ||
| 70 : | catch (Exception) | ||
| 71 : | { | ||
| 72 : | Logger.Log.Error("Failed to load [Amazon] section from config file " + AssetServer.CONFIG_FILE); | ||
| 73 : | return; | ||
| 74 : | } | ||
| 75 : | |||
| 76 : | try | ||
| 77 : | { | ||
| 78 : | s3.CreateBucket(bucketName); | ||
| 79 : | BucketAccess access = s3.QueryBucket(bucketName); | ||
| 80 : | if (access == BucketAccess.Accessible) | ||
| 81 : | { | ||
| 82 : | string bucketHost = bucketName + ".s3.amazonaws.com"; | ||
| 83 : | Logger.Log.Info("Using Amazon S3 bucket http://" + bucketHost); | ||
| 84 : | |||
| 85 : | if (useCloudFront) | ||
| 86 : | { | ||
| 87 : | CloudFrontService cf = new CloudFrontService(); | ||
| 88 : | cf.AccessKeyID = s3.AccessKeyID; | ||
| 89 : | cf.SecretAccessKey = s3.SecretAccessKey; | ||
| 90 : | |||
| 91 : | // List all of the distributions associated with our account | ||
| 92 : | foreach (Distribution distribution in cf.GetAllDistributions()) | ||
| 93 : | { | ||
| 94 : | // Check if any of the distributions are tied to the current S3 bucket | ||
| 95 : | if (distribution.Config.Origin == bucketHost && distribution.Config.Enabled) | ||
| 96 : | { | ||
| 97 : | if (WaitForCloudFront(cf, distribution)) | ||
| 98 : | cloudFrontHost = distribution.DomainName; | ||
| 99 : | break; | ||
| 100 : | } | ||
| 101 : | } | ||
| 102 : | |||
| 103 : | if (cloudFrontHost == null) | ||
| 104 : | { | ||
| 105 : | // No valid distribution was found, create a new one | ||
| 106 : | DistributionConfig config = new DistributionConfig(); | ||
| 107 : | config.Origin = bucketHost; | ||
| 108 : | config.Enabled = true; | ||
| 109 : | Distribution distribution = cf.CreateDistribution(config); | ||
| 110 : | |||
| 111 : | if (WaitForCloudFront(cf, distribution)) | ||
| 112 : | cloudFrontHost = distribution.DomainName; | ||
| 113 : | else | ||
| 114 : | Logger.Log.Error("Failed to create Amazon CloudFront distribution"); | ||
| 115 : | } | ||
| 116 : | |||
| 117 : | if (cloudFrontHost != null) | ||
| 118 : | { | ||
| 119 : | Logger.Log.Info("Using Amazon CloudFront distribution http://" + cloudFrontHost); | ||
| 120 : | } | ||
| 121 : | } | ||
| 122 : | } | ||
| 123 : | else | ||
| 124 : | { | ||
| 125 : | Logger.Log.ErrorFormat("Cannot use Amazon S3 bucket {0}: {1}", bucketName, access); | ||
| 126 : | } | ||
| 127 : | } | ||
| 128 : | catch (Exception ex) | ||
| 129 : | { | ||
| 130 : | Logger.Log.ErrorFormat("Failed to create/use Amazon S3 bucket {0}: {1}", bucketName, ex.Message); | ||
| 131 : | } | ||
| 132 : | } | ||
| 133 : | |||
| 134 : | public void Stop() | ||
| 135 : | { | ||
| 136 : | } | ||
| 137 : | |||
| 138 : | jhurliman | 69 | public StorageResponse TryFetchMetadata(UUID assetID, out Metadata metadata) |
| 139 : | jhurliman | 49 | { |
| 140 : | metadata = null; | ||
| 141 : | StorageResponse ret; | ||
| 142 : | |||
| 143 : | jhurliman | 69 | GetObjectRequest request = new GetObjectRequest(s3, bucketName, assetID.ToString(), true); |
| 144 : | jhurliman | 49 | |
| 145 : | jhurliman | 69 | try |
| 146 : | { | ||
| 147 : | using (GetObjectResponse response = request.GetResponse()) | ||
| 148 : | jhurliman | 49 | { |
| 149 : | jhurliman | 69 | metadata = new Metadata(); |
| 150 : | metadata.CreationDate = response.LastModified; | ||
| 151 : | metadata.Description = response.Metadata.Get("description"); | ||
| 152 : | metadata.ID = assetID; | ||
| 153 : | metadata.Name = response.Metadata.Get("name"); | ||
| 154 : | metadata.SHA1 = OpenMetaverse.Utils.HexStringToBytes(response.Metadata.Get("sha1"), false); | ||
| 155 : | Boolean.TryParse(response.Metadata.Get("temporary"), out metadata.Temporary); | ||
| 156 : | metadata.ContentType = response.ContentType; | ||
| 157 : | jhurliman | 49 | |
| 158 : | jhurliman | 69 | // Return the CloudFront URL if enabled |
| 159 : | if (cloudFrontHost != null) | ||
| 160 : | metadata.Methods["data"] = new Uri(String.Format("http://{0}/{1}", cloudFrontHost, assetID)); | ||
| 161 : | jhurliman | 49 | else |
| 162 : | jhurliman | 69 | metadata.Methods["data"] = new Uri(s3.GetUrl(bucketName, assetID.ToString())); |
| 163 : | |||
| 164 : | ret = StorageResponse.Success; | ||
| 165 : | jhurliman | 49 | } |
| 166 : | } | ||
| 167 : | jhurliman | 69 | catch (WebException ex) |
| 168 : | jhurliman | 49 | { |
| 169 : | jhurliman | 69 | if (ex.Response != null && (ex.Response as HttpWebResponse).StatusCode == HttpStatusCode.NotFound) |
| 170 : | { | ||
| 171 : | ret = StorageResponse.NotFound; | ||
| 172 : | } | ||
| 173 : | else | ||
| 174 : | { | ||
| 175 : | Logger.Log.WarnFormat("Failed fetching metadata for {0}: {1}", assetID, ex.Message); | ||
| 176 : | ret = StorageResponse.Failure; | ||
| 177 : | } | ||
| 178 : | jhurliman | 49 | } |
| 179 : | |||
| 180 : | jhurliman | 69 | server.MetricsProvider.LogAssetMetadataFetch(EXTENSION_NAME, ret, assetID, DateTime.Now); |
| 181 : | jhurliman | 49 | return ret; |
| 182 : | } | ||
| 183 : | |||
| 184 : | jhurliman | 69 | public StorageResponse TryFetchData(UUID assetID, out byte[] assetData) |
| 185 : | jhurliman | 49 | { |
| 186 : | assetData = null; | ||
| 187 : | long contentLength = 0; | ||
| 188 : | string contentType; | ||
| 189 : | StorageResponse ret; | ||
| 190 : | |||
| 191 : | jhurliman | 69 | try |
| 192 : | jhurliman | 49 | { |
| 193 : | jhurliman | 69 | using (Stream stream = s3.GetObjectStream(bucketName, assetID.ToString(), out contentLength, out contentType)) |
| 194 : | jhurliman | 49 | { |
| 195 : | jhurliman | 69 | assetData = new byte[contentLength]; |
| 196 : | jhurliman | 49 | |
| 197 : | jhurliman | 69 | int pos = 0; |
| 198 : | while (pos < contentLength) | ||
| 199 : | pos += stream.Read(assetData, pos, (int)contentLength - pos); | ||
| 200 : | } | ||
| 201 : | jhurliman | 49 | |
| 202 : | jhurliman | 69 | ret = StorageResponse.Success; |
| 203 : | } | ||
| 204 : | catch (WebException ex) | ||
| 205 : | { | ||
| 206 : | if (ex.Response != null && (ex.Response as HttpWebResponse).StatusCode == HttpStatusCode.NotFound) | ||
| 207 : | { | ||
| 208 : | ret = StorageResponse.NotFound; | ||
| 209 : | jhurliman | 49 | } |
| 210 : | jhurliman | 69 | else |
| 211 : | jhurliman | 49 | { |
| 212 : | jhurliman | 69 | Logger.Log.WarnFormat("Failed fetching data for {0}: {1}", assetID, ex.Message); |
| 213 : | ret = StorageResponse.Failure; | ||
| 214 : | jhurliman | 49 | } |
| 215 : | } | ||
| 216 : | |||
| 217 : | jhurliman | 69 | server.MetricsProvider.LogAssetDataFetch(EXTENSION_NAME, ret, assetID, (int)contentLength, DateTime.Now); |
| 218 : | jhurliman | 49 | return ret; |
| 219 : | } | ||
| 220 : | |||
| 221 : | jhurliman | 69 | public StorageResponse TryFetchDataMetadata(UUID assetID, out Metadata metadata, out byte[] assetData) |
| 222 : | jhurliman | 49 | { |
| 223 : | metadata = null; | ||
| 224 : | |||
| 225 : | jhurliman | 69 | StorageResponse response = TryFetchData(assetID, out assetData); |
| 226 : | jhurliman | 49 | if (response == StorageResponse.Success) |
| 227 : | jhurliman | 69 | response = TryFetchMetadata(assetID, out metadata); |
| 228 : | jhurliman | 49 | |
| 229 : | return response; | ||
| 230 : | } | ||
| 231 : | |||
| 232 : | jhurliman | 69 | public StorageResponse TryCreateAsset(Metadata metadata, byte[] assetData, out UUID assetID) |
| 233 : | jhurliman | 49 | { |
| 234 : | assetID = metadata.ID = UUID.Random(); | ||
| 235 : | jhurliman | 69 | return TryCreateAsset(metadata, assetData); |
| 236 : | jhurliman | 49 | } |
| 237 : | |||
| 238 : | jhurliman | 69 | public StorageResponse TryCreateAsset(Metadata metadata, byte[] assetData) |
| 239 : | jhurliman | 49 | { |
| 240 : | StorageResponse ret; | ||
| 241 : | |||
| 242 : | jhurliman | 69 | // Calculate the MD5 to compare against what AmazonS3 returns |
| 243 : | string md5String = BitConverter.ToString(OpenMetaverse.Utils.MD5(assetData)).Replace("-", String.Empty); | ||
| 244 : | jhurliman | 49 | |
| 245 : | jhurliman | 69 | AddObjectRequest request = new AddObjectRequest(s3, bucketName, metadata.ID.ToString()); |
| 246 : | request.CannedAcl = CannedAcl.PublicRead; | ||
| 247 : | request.ReadWriteTimeout = 1000 * 60; | ||
| 248 : | request.ContentLength = assetData.Length; | ||
| 249 : | request.ContentType = metadata.ContentType; | ||
| 250 : | //request.ContentDisposition = String.Format("attachment; filename={0}", metadata.ID); | ||
| 251 : | request.Metadata.Add("name", metadata.Name); | ||
| 252 : | request.Metadata.Add("description", metadata.Description); | ||
| 253 : | request.Metadata.Add("temporary", metadata.Temporary.ToString()); | ||
| 254 : | request.Metadata.Add("sha1", BitConverter.ToString(metadata.SHA1).Replace("-", String.Empty)); | ||
| 255 : | jhurliman | 49 | |
| 256 : | jhurliman | 69 | try |
| 257 : | { | ||
| 258 : | using (Stream stream = request.GetRequestStream()) | ||
| 259 : | jhurliman | 49 | { |
| 260 : | jhurliman | 69 | stream.Write(assetData, 0, assetData.Length); |
| 261 : | stream.Flush(); | ||
| 262 : | } | ||
| 263 : | jhurliman | 49 | |
| 264 : | jhurliman | 69 | AddObjectResponse response = request.GetResponse(); |
| 265 : | response.Close(); | ||
| 266 : | jhurliman | 49 | |
| 267 : | jhurliman | 69 | if (md5String.Equals(response.ETag.Trim('"'), StringComparison.OrdinalIgnoreCase)) |
| 268 : | { | ||
| 269 : | ret = StorageResponse.Success; | ||
| 270 : | jhurliman | 49 | } |
| 271 : | jhurliman | 69 | else |
| 272 : | jhurliman | 49 | { |
| 273 : | jhurliman | 69 | Logger.Log.ErrorFormat( |
| 274 : | "MD5 of uploaded asset ({0}) does not match match asset MD5 ({1}), deleting upload", | ||
| 275 : | response.ETag.Trim('"').ToLower(), md5String); | ||
| 276 : | |||
| 277 : | // Delete the failed upload | ||
| 278 : | s3.DeleteObject(bucketName, metadata.ID.ToString()); | ||
| 279 : | |||
| 280 : | jhurliman | 49 | ret = StorageResponse.Failure; |
| 281 : | } | ||
| 282 : | } | ||
| 283 : | jhurliman | 69 | catch (WebException ex) |
| 284 : | jhurliman | 49 | { |
| 285 : | jhurliman | 69 | Logger.Log.WarnFormat("Failed uploading asset {0}: {1}", metadata.ID, ex.Message); |
| 286 : | ret = StorageResponse.Failure; | ||
| 287 : | jhurliman | 49 | } |
| 288 : | |||
| 289 : | jhurliman | 69 | server.MetricsProvider.LogAssetCreate(EXTENSION_NAME, ret, metadata.ID, assetData.Length, DateTime.Now); |
| 290 : | jhurliman | 49 | return ret; |
| 291 : | } | ||
| 292 : | |||
| 293 : | jhurliman | 69 | public int ForEach(Action<Metadata> action, int start, int count) |
| 294 : | jhurliman | 49 | { |
| 295 : | string nextMarker = null; | ||
| 296 : | int pos = 0; | ||
| 297 : | int rowCount = 0; | ||
| 298 : | |||
| 299 : | #region Move To Beginning | ||
| 300 : | // S3 does not allow us to say "start at asset 14", but we can say | ||
| 301 : | // "start at asset N", where N is a marker. We first find the | ||
| 302 : | // marker for the asset at our starting position | ||
| 303 : | try | ||
| 304 : | { | ||
| 305 : | while (pos < start) | ||
| 306 : | { | ||
| 307 : | ListObjectsArgs args = new ListObjectsArgs(); | ||
| 308 : | args.Marker = nextMarker; | ||
| 309 : | args.MaxKeys = start - pos; | ||
| 310 : | args.Delimiter = "/"; | ||
| 311 : | ListObjectsRequest request = new ListObjectsRequest(s3, bucketName, args); | ||
| 312 : | |||
| 313 : | ListObjectsResponse response = request.GetResponse(); | ||
| 314 : | response.Close(); | ||
| 315 : | |||
| 316 : | foreach (ObjectEntry entry in response.Entries) | ||
| 317 : | ++pos; | ||
| 318 : | |||
| 319 : | if (response.IsTruncated) | ||
| 320 : | { | ||
| 321 : | nextMarker = response.NextMarker; | ||
| 322 : | } | ||
| 323 : | else | ||
| 324 : | { | ||
| 325 : | // We reached the end of the asset list without getting to the | ||
| 326 : | // start position | ||
| 327 : | Logger.Log.WarnFormat( | ||
| 328 : | "AmazonS3: Iterated through the entire asset list ({0} counted) without reaching start position {1}", | ||
| 329 : | pos, start); | ||
| 330 : | return 0; | ||
| 331 : | } | ||
| 332 : | } | ||
| 333 : | } | ||
| 334 : | catch (WebException ex) | ||
| 335 : | { | ||
| 336 : | Logger.Log.Warn("Failed fetching AmazonS3 asset list: " + ex.Message); | ||
| 337 : | return 0; | ||
| 338 : | } | ||
| 339 : | #endregion Move To Beginning | ||
| 340 : | |||
| 341 : | // nextMarker is now set to the position of start, or null (indicating starting at the beginning) | ||
| 342 : | Logger.Log.DebugFormat("Starting AmazonS3 ForEach(), start={0}, pos={1}, nextMarker={2}", start, pos, nextMarker); | ||
| 343 : | |||
| 344 : | try | ||
| 345 : | { | ||
| 346 : | ListObjectsArgs args = new ListObjectsArgs(); | ||
| 347 : | args.Marker = nextMarker; | ||
| 348 : | args.MaxKeys = count; | ||
| 349 : | args.Delimiter = "/"; | ||
| 350 : | ListObjectsRequest request = new ListObjectsRequest(s3, bucketName, args); | ||
| 351 : | |||
| 352 : | ListObjectsResponse response = request.GetResponse(); | ||
| 353 : | response.Close(); | ||
| 354 : | |||
| 355 : | foreach (ObjectEntry entry in response.Entries) | ||
| 356 : | { | ||
| 357 : | UUID assetID; | ||
| 358 : | Metadata metadata; | ||
| 359 : | if (UUID.TryParse(entry.Key, out assetID) && | ||
| 360 : | jhurliman | 69 | TryFetchMetadata(assetID, out metadata) == StorageResponse.Success) |
| 361 : | jhurliman | 49 | { |
| 362 : | action(metadata); | ||
| 363 : | ++rowCount; | ||
| 364 : | } | ||
| 365 : | } | ||
| 366 : | } | ||
| 367 : | catch (WebException ex) | ||
| 368 : | { | ||
| 369 : | Logger.Log.Warn("Failed fetching AmazonS3 asset list: " + ex.Message); | ||
| 370 : | } | ||
| 371 : | |||
| 372 : | return rowCount; | ||
| 373 : | } | ||
| 374 : | |||
| 375 : | static bool WaitForCloudFront(CloudFrontService cf, Distribution distribution) | ||
| 376 : | { | ||
| 377 : | while (distribution.Status == DistributionStatus.InProgress) | ||
| 378 : | { | ||
| 379 : | Logger.Log.Debug("Distribution is being deployed, waiting..."); | ||
| 380 : | System.Threading.Thread.Sleep(1000 * 10); | ||
| 381 : | |||
| 382 : | try | ||
| 383 : | { | ||
| 384 : | distribution = cf.GetDistributionInfo(distribution.ID); | ||
| 385 : | } | ||
| 386 : | catch (Exception ex) | ||
| 387 : | { | ||
| 388 : | Logger.Log.Error("Error waiting for CloudFront distribution to come online: " + ex.Message); | ||
| 389 : | return false; | ||
| 390 : | } | ||
| 391 : | } | ||
| 392 : | |||
| 393 : | return true; | ||
| 394 : | } | ||
| 395 : | } | ||
| 396 : | } |
| ViewVC Help | |
| Powered by ViewVC 1.0.0 |
