diff --git a/.gitignore b/.gitignore index 592d6c4..d5d599a 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ DerivedData *.ipa # Swift Package Manager -*/.build +.build Package.resolved # Mint package manager diff --git a/GoogleDataTransport.podspec b/GoogleDataTransport.podspec index c7ac379..cc637ef 100644 --- a/GoogleDataTransport.podspec +++ b/GoogleDataTransport.podspec @@ -39,7 +39,9 @@ Shared library for iOS SDK data transport needs. s.libraries = ['z'] + s.dependency 'GoogleUtilities/Environment', '~> 7.1' s.dependency 'nanopb', '~> 2.30907.0' + s.dependency 'PromisesObjC', '~> 1.2' header_search_paths = { 'HEADER_SEARCH_PATHS' => '"${PODS_TARGET_SRCROOT}/"' @@ -161,9 +163,14 @@ Shared library for iOS SDK data transport needs. test_spec.scheme = { :code_coverage => true } test_spec.platforms = {:ios => ios_deployment_target, :osx => osx_deployment_target, :tvos => tvos_deployment_target} test_spec.requires_app_host = false - test_spec.source_files = ['GoogleDataTransport/GDTCCTTests/Integration/**/*.{h,m}'] + common_cct_test_sources + test_spec.source_files = [ + 'GoogleDataTransport/GDTCCTTests/Integration/**/*.{h,m}', + 'GoogleDataTransport/GDTCCTTests/Unit/TestServer/GDTCCTTestServer.{h,m}', + 'GoogleDataTransport/GDTCORTests/Common/Categories/GDTCORFlatFileStorage+Testing.{h,m}' + ] + common_cct_test_sources test_spec.resources = ['GoogleDataTransport/GDTCCTTests/Data/**/*'] test_spec.pod_target_xcconfig = header_search_paths + test_spec.dependency 'GCDWebServer' end # Monkey test specs, only enabled for development. diff --git a/GoogleDataTransport/GDTCCTLibrary/GDTCCTUploadOperation.m b/GoogleDataTransport/GDTCCTLibrary/GDTCCTUploadOperation.m new file mode 100644 index 0000000..fdabc07 --- /dev/null +++ b/GoogleDataTransport/GDTCCTLibrary/GDTCCTUploadOperation.m @@ -0,0 +1,526 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "GoogleDataTransport/GDTCCTLibrary/Private/GDTCCTUploadOperation.h" + +#if __has_include() +#import +#else +#import "FBLPromises.h" +#endif + +#import "GoogleDataTransport/GDTCORLibrary/Internal/GDTCORPlatform.h" +#import "GoogleDataTransport/GDTCORLibrary/Internal/GDTCORRegistrar.h" +#import "GoogleDataTransport/GDTCORLibrary/Internal/GDTCORStorageProtocol.h" +#import "GoogleDataTransport/GDTCORLibrary/Private/GDTCORUploadBatch.h" +#import "GoogleDataTransport/GDTCORLibrary/Public/GoogleDataTransport/GDTCORConsoleLogger.h" +#import "GoogleDataTransport/GDTCORLibrary/Public/GoogleDataTransport/GDTCOREvent.h" + +#import +#import +#import + +#import +#import +#import "GoogleDataTransport/GDTCCTLibrary/Private/GDTCCTCompressionHelper.h" +#import "GoogleDataTransport/GDTCCTLibrary/Private/GDTCCTNanopbHelpers.h" + +#import "GoogleDataTransport/GDTCCTLibrary/Protogen/nanopb/cct.nanopb.h" + +NS_ASSUME_NONNULL_BEGIN + +#ifdef GDTCOR_VERSION +#define STR(x) STR_EXPAND(x) +#define STR_EXPAND(x) #x +static NSString *const kGDTCCTSupportSDKVersion = @STR(GDTCOR_VERSION); +#else +static NSString *const kGDTCCTSupportSDKVersion = @"UNKNOWN"; +#endif // GDTCOR_VERSION + +typedef void (^GDTCCTUploaderURLTaskCompletion)(NSNumber *batchID, + NSSet *_Nullable events, + NSData *_Nullable data, + NSURLResponse *_Nullable response, + NSError *_Nullable error); + +typedef void (^GDTCCTUploaderEventBatchBlock)(NSNumber *_Nullable batchID, + NSSet *_Nullable events); + +@interface GDTCCTUploadOperation () + +@property(nonatomic, readonly) GDTCORTarget target; +@property(nonatomic, readonly) GDTCORUploadConditions conditions; +@property(nonatomic, readonly) NSURL *uploadURL; +@property(nonatomic, readonly) id storage; +@property(nonatomic, readonly) id metadataProvider; + +@property(nonatomic, readwrite, getter=isExecuting) BOOL executing; +@property(nonatomic, readwrite, getter=isFinished) BOOL finished; + +@property(nonatomic, readwrite) BOOL uploadAttempted; + +@end + +@implementation GDTCCTUploadOperation + +- (instancetype)initWithTarget:(GDTCORTarget)target + conditions:(GDTCORUploadConditions)conditions + uploadURL:(NSURL *)uploadURL + queue:(dispatch_queue_t)queue + storage:(id)storage + metadataProvider:(id)metadataProvider { + self = [super init]; + if (self) { + _uploaderQueue = queue; + NSURLSessionConfiguration *config = [NSURLSessionConfiguration defaultSessionConfiguration]; + _uploaderSession = [NSURLSession sessionWithConfiguration:config + delegate:self + delegateQueue:nil]; + _target = target; + _conditions = conditions; + _uploadURL = uploadURL; + _storage = storage; + _metadataProvider = metadataProvider; + } + return self; +} + +- (void)uploadTarget:(GDTCORTarget)target withConditions:(GDTCORUploadConditions)conditions { + __block GDTCORBackgroundIdentifier backgroundTaskID = GDTCORBackgroundIdentifierInvalid; + + dispatch_block_t backgroundTaskCompletion = ^{ + // End the background task if there was one. + if (backgroundTaskID != GDTCORBackgroundIdentifierInvalid) { + [[GDTCORApplication sharedApplication] endBackgroundTask:backgroundTaskID]; + backgroundTaskID = GDTCORBackgroundIdentifierInvalid; + } + }; + + backgroundTaskID = [[GDTCORApplication sharedApplication] + beginBackgroundTaskWithName:@"GDTCCTUploader-upload" + expirationHandler:^{ + if (backgroundTaskID != GDTCORBackgroundIdentifierInvalid) { + // Cancel the upload and complete delivery. + [self.currentTask cancel]; + + // End the background task. + backgroundTaskCompletion(); + } + }]; + + id storage = self.storage; + + // 1. Check if the conditions for the target are suitable. + [self isReadyToUploadTarget:target conditions:conditions] + .thenOn(self.uploaderQueue, + ^id(id result) { + // 2. Remove previously attempted batches + return [storage removeAllBatchesForTarget:target deleteEvents:NO]; + }) + .thenOn(self.uploaderQueue, + ^FBLPromise *(id result) { + // There may be a big amount of events stored, so creating a batch may be an + // expensive operation. + + // 3. Do a lightweight check if there are any events for the target first to + // finish early if there are no. + return [storage hasEventsForTarget:target]; + }) + .validateOn(self.uploaderQueue, + ^BOOL(NSNumber *hasEvents) { + // Stop operation if there are no events to upload. + return hasEvents.boolValue; + }) + .thenOn(self.uploaderQueue, + ^FBLPromise *(id result) { + if (self.isCancelled) { + return nil; + } + + // 4. Fetch events to upload. + GDTCORStorageEventSelector *eventSelector = [self eventSelectorTarget:target + withConditions:conditions]; + return [storage batchWithEventSelector:eventSelector + batchExpiration:[NSDate dateWithTimeIntervalSinceNow:600]]; + }) + .validateOn(self.uploaderQueue, + ^BOOL(GDTCORUploadBatch *batch) { + // 5. Validate batch. + return batch.batchID != nil && batch.events.count > 0; + }) + .thenOn(self.uploaderQueue, + ^FBLPromise *(GDTCORUploadBatch *batch) { + // A non-empty batch has been created, consider it as an upload attempt. + self.uploadAttempted = YES; + + // 6. Perform upload URL request. + return [self sendURLRequestWithBatch:batch target:target storage:storage]; + }) + .thenOn(self.uploaderQueue, + ^id(id result) { + // 7. Finish operation. + [self finishOperation]; + return nil; + }) + .catchOn(self.uploaderQueue, ^(NSError *error) { + // TODO: Maybe report the error to the client. + [self finishOperation]; + }); +} + +#pragma mark - Upload implementation details + +/** Sends URL request to upload the provided batch and handle the response. */ +- (FBLPromise *)sendURLRequestWithBatch:(GDTCORUploadBatch *)batch + target:(GDTCORTarget)target + storage:(id)storage { + NSNumber *batchID = batch.batchID; + + // 1. Send URL request. + return [self sendURLRequestWithBatch:batch target:target] + .thenOn( + self.uploaderQueue, + ^FBLPromise *(GULURLSessionDataResponse *response) { + // 2. Parse response and update the next upload time if can. + [self updateNextUploadTimeWithResponse:response forTarget:target]; + + // 3. Cleanup batch. + + // Only retry if one of these codes is returned: + // 429 - Too many requests; + // 503 - Service unavailable. + NSInteger statusCode = response.HTTPResponse.statusCode; + if (statusCode == 429 || statusCode == 503) { + // Move the events back to the main storage to be uploaded on the next attempt. + return [storage removeBatchWithID:batchID deleteEvents:NO]; + } else { + if (statusCode >= 200 && statusCode <= 300) { + GDTCORLogDebug(@"CCT: batch %@ delivered", batchID); + } else { + GDTCORLogDebug( + @"CCT: batch %@ was rejected by the server and will be deleted with all events", + batchID); + } + + // The events are either delivered or unrecoverable broken, so remove the batch with + // events. + return [storage removeBatchWithID:batch.batchID deleteEvents:YES]; + } + }) + .recoverOn(self.uploaderQueue, ^id(NSError *error) { + // In the case of a network error move the events back to the main storage to be uploaded on + // the next attempt. + return [storage removeBatchWithID:batchID deleteEvents:NO]; + }); +} + +/** Composes and sends URL request. */ +- (FBLPromise *)sendURLRequestWithBatch:(GDTCORUploadBatch *)batch + target:(GDTCORTarget)target { + return [FBLPromise + onQueue:self.uploaderQueue + do:^NSURLRequest * { + // 1. Prepare URL request. + NSData *requestProtoData = [self constructRequestProtoWithEvents:batch.events]; + NSData *gzippedData = [GDTCCTCompressionHelper gzippedData:requestProtoData]; + BOOL usingGzipData = + gzippedData != nil && gzippedData.length < requestProtoData.length; + NSData *dataToSend = usingGzipData ? gzippedData : requestProtoData; + NSURLRequest *request = [self constructRequestForTarget:target data:dataToSend]; + GDTCORLogDebug(@"CTT: request containing %lu events for batch: %@ for target: " + @"%ld created: %@", + (unsigned long)batch.events.count, batch.batchID, (long)target, + request); + return request; + }] + .thenOn(self.uploaderQueue, + ^FBLPromise *(NSURLRequest *request) { + // 2. Send URL request. + return [self.uploaderSession gul_dataTaskPromiseWithRequest:request]; + }); +} + +/** Parses server response and update next upload time for the specified target based on it. */ +- (void)updateNextUploadTimeWithResponse:(GULURLSessionDataResponse *)response + forTarget:(GDTCORTarget)target { + GDTCORClock *futureUploadTime; + if (response.HTTPBody) { + NSError *decodingError; + gdt_cct_LogResponse logResponse = GDTCCTDecodeLogResponse(response.HTTPBody, &decodingError); + if (!decodingError && logResponse.has_next_request_wait_millis) { + GDTCORLogDebug(@"CCT: The backend responded asking to not upload for %lld millis from now.", + logResponse.next_request_wait_millis); + futureUploadTime = + [GDTCORClock clockSnapshotInTheFuture:logResponse.next_request_wait_millis]; + } else if (decodingError) { + GDTCORLogDebug(@"There was a response decoding error: %@", decodingError); + } + pb_release(gdt_cct_LogResponse_fields, &logResponse); + } + + if (!futureUploadTime) { + GDTCORLogDebug(@"%@", @"CCT: The backend response failed to parse, so the next request " + @"won't occur until 15 minutes from now"); + // 15 minutes from now. + futureUploadTime = [GDTCORClock clockSnapshotInTheFuture:15 * 60 * 1000]; + } + + [self.metadataProvider setNextUploadTime:futureUploadTime forTarget:target]; +} + +#pragma mark - Private helper methods + +/** @return A resolved promise if is ready and a rejected promise if not. */ +- (FBLPromise *)isReadyToUploadTarget:(GDTCORTarget)target + conditions:(GDTCORUploadConditions)conditions { + FBLPromise *promise = [FBLPromise pendingPromise]; + if ([self readyToUploadTarget:target conditions:conditions]) { + [promise fulfill:[NSNull null]]; + } else { + // TODO: Do we need a more comprehensive message here? + [promise reject:[self genericRejectedPromiseErrorWithReason:@"Is not ready."]]; + } + return promise; +} + +// TODO: Move to a separate class/extension/file. +- (NSError *)genericRejectedPromiseErrorWithReason:(NSString *)reason { + return [NSError errorWithDomain:@"GDTCCTUploader" + code:-1 + userInfo:@{NSLocalizedFailureReasonErrorKey : reason}]; +} + +/** */ +- (BOOL)readyToUploadTarget:(GDTCORTarget)target conditions:(GDTCORUploadConditions)conditions { + // Not ready to upload with no network connection. + // TODO: Reconsider using reachability to prevent an upload attempt. + // See https://developer.apple.com/videos/play/wwdc2019/712/ (49:40) for more details. + if (conditions & GDTCORUploadConditionNoNetwork) { + GDTCORLogDebug(@"%@", @"CCT: Not ready to upload without a network connection."); + return NO; + } + + // Upload events when there are with no additional conditions for kGDTCORTargetCSH. + if (target == kGDTCORTargetCSH) { + GDTCORLogDebug(@"%@", @"CCT: kGDTCORTargetCSH events are allowed to be " + @"uploaded straight away."); + return YES; + } + + if (target == kGDTCORTargetINT) { + GDTCORLogDebug(@"%@", @"CCT: kGDTCORTargetINT events are allowed to be " + @"uploaded straight away."); + return YES; + } + + // Upload events with no additional conditions if high priority. + if ((conditions & GDTCORUploadConditionHighPriority) == GDTCORUploadConditionHighPriority) { + GDTCORLogDebug(@"%@", @"CCT: a high priority event is allowing an upload"); + return YES; + } + + // Check next upload time for the target. + BOOL isAfterNextUploadTime = YES; + + // TODO: Should other targets respect the next upload time as well? + // Only kGDTCORTargetCCT and kGDTCORTargetFLL respect next upload time now. + if (target == kGDTCORTargetCCT || target == kGDTCORTargetFLL) { + GDTCORClock *nextUploadTime = [self.metadataProvider nextUploadTimeForTarget:target]; + if (nextUploadTime) { + isAfterNextUploadTime = [[GDTCORClock snapshot] isAfter:nextUploadTime]; + } + } + + if (isAfterNextUploadTime) { + GDTCORLogDebug(@"CCT: can upload to target %ld because the request wait time has transpired", + (long)target); + } else { + GDTCORLogDebug(@"CCT: can't upload to target %ld because the backend asked to wait", + (long)target); + } + + return isAfterNextUploadTime; +} + +/** Constructs data given an upload package. + * + * @param events The events used to construct the request proto bytes. + * @return Proto bytes representing a gdt_cct_LogRequest object. + */ +- (nonnull NSData *)constructRequestProtoWithEvents:(NSSet *)events { + // Segment the log events by log type. + NSMutableDictionary *> *logMappingIDToLogSet = + [[NSMutableDictionary alloc] init]; + [events enumerateObjectsUsingBlock:^(GDTCOREvent *_Nonnull event, BOOL *_Nonnull stop) { + NSMutableSet *logSet = logMappingIDToLogSet[event.mappingID]; + logSet = logSet ? logSet : [[NSMutableSet alloc] init]; + [logSet addObject:event]; + logMappingIDToLogSet[event.mappingID] = logSet; + }]; + + gdt_cct_BatchedLogRequest batchedLogRequest = + GDTCCTConstructBatchedLogRequest(logMappingIDToLogSet); + + NSData *data = GDTCCTEncodeBatchedLogRequest(&batchedLogRequest); + pb_release(gdt_cct_BatchedLogRequest_fields, &batchedLogRequest); + return data ? data : [[NSData alloc] init]; +} + +/** Constructs a request to FLL given a URL and request body data. + * + * @param target The target backend to send the request to. + * @param data The request body data. + * @return A new NSURLRequest ready to be sent to FLL. + */ +- (nullable NSURLRequest *)constructRequestForTarget:(GDTCORTarget)target data:(NSData *)data { + if (data == nil || data.length == 0) { + GDTCORLogDebug(@"There was no data to construct a request for target %ld.", (long)target); + return nil; + } + NSURL *URL = self.uploadURL; + NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:URL]; + NSString *targetString; + switch (target) { + case kGDTCORTargetCCT: + targetString = @"cct"; + break; + + case kGDTCORTargetFLL: + targetString = @"fll"; + break; + + case kGDTCORTargetCSH: + targetString = @"csh"; + break; + case kGDTCORTargetINT: + targetString = @"int"; + break; + + default: + targetString = @"unknown"; + break; + } + NSString *userAgent = + [NSString stringWithFormat:@"datatransport/%@ %@support/%@ apple/", kGDTCORVersion, + targetString, kGDTCCTSupportSDKVersion]; + + [request setValue:[self.metadataProvider APIKeyForTarget:target] + forHTTPHeaderField:@"X-Goog-Api-Key"]; + + if ([GDTCCTCompressionHelper isGzipped:data]) { + [request setValue:@"gzip" forHTTPHeaderField:@"Content-Encoding"]; + } + [request setValue:@"application/x-protobuf" forHTTPHeaderField:@"Content-Type"]; + [request setValue:@"gzip" forHTTPHeaderField:@"Accept-Encoding"]; + [request setValue:userAgent forHTTPHeaderField:@"User-Agent"]; + request.HTTPMethod = @"POST"; + [request setHTTPBody:data]; + return request; +} + +/** */ +- (nullable GDTCORStorageEventSelector *)eventSelectorTarget:(GDTCORTarget)target + withConditions:(GDTCORUploadConditions)conditions { + if ((conditions & GDTCORUploadConditionHighPriority) == GDTCORUploadConditionHighPriority) { + return [GDTCORStorageEventSelector eventSelectorForTarget:target]; + } + NSMutableSet *qosTiers = [[NSMutableSet alloc] init]; + if (conditions & GDTCORUploadConditionWifiData) { + [qosTiers addObjectsFromArray:@[ + @(GDTCOREventQoSFast), @(GDTCOREventQoSWifiOnly), @(GDTCOREventQosDefault), + @(GDTCOREventQoSTelemetry), @(GDTCOREventQoSUnknown) + ]]; + } + if (conditions & GDTCORUploadConditionMobileData) { + [qosTiers addObjectsFromArray:@[ @(GDTCOREventQoSFast), @(GDTCOREventQosDefault) ]]; + } + + return [[GDTCORStorageEventSelector alloc] initWithTarget:target + eventIDs:nil + mappingIDs:nil + qosTiers:qosTiers]; +} + +#pragma mark - NSURLSessionDelegate + +- (void)URLSession:(NSURLSession *)session + task:(NSURLSessionTask *)task + willPerformHTTPRedirection:(NSHTTPURLResponse *)response + newRequest:(NSURLRequest *)request + completionHandler:(void (^)(NSURLRequest *_Nullable))completionHandler { + if (!completionHandler) { + return; + } + if (response.statusCode == 302 || response.statusCode == 301) { + // TODO: Take a redirect URL from the response. + // if ([request.URL isEqual:[self serverURLForTarget:kGDTCORTargetFLL]]) { + // NSURLRequest *newRequest = [self constructRequestForTarget:kGDTCORTargetCCT + // data:task.originalRequest.HTTPBody]; + // completionHandler(newRequest); + // } + } else { + completionHandler(request); + } +} + +#pragma mark - NSOperation methods + +@synthesize executing = _executing; +@synthesize finished = _finished; + +- (BOOL)isAsynchronous { + return YES; +} + +- (void)startOperation { + [self willChangeValueForKey:@"isExecuting"]; + [self willChangeValueForKey:@"isFinished"]; + _executing = YES; + _finished = NO; + [self didChangeValueForKey:@"isExecuting"]; + [self didChangeValueForKey:@"isFinished"]; +} + +- (void)finishOperation { + [self willChangeValueForKey:@"isExecuting"]; + [self willChangeValueForKey:@"isFinished"]; + _executing = NO; + _finished = YES; + [self didChangeValueForKey:@"isExecuting"]; + [self didChangeValueForKey:@"isFinished"]; +} + +- (void)main { + [self startOperation]; + + GDTCORLogDebug(@"Upload operation started: %@", self); + [self uploadTarget:self.target withConditions:self.conditions]; +} + +- (void)cancel { + GDTCORLogDebug(@"Upload operation cancelled: %@", self); + [super cancel]; + + // If the operation hasn't been started we can set `isFinished = YES` straight away. + if (!_executing) { + _executing = NO; + _finished = YES; + } +} + +@end + +NS_ASSUME_NONNULL_END diff --git a/GoogleDataTransport/GDTCCTLibrary/GDTCCTUploader.m b/GoogleDataTransport/GDTCCTLibrary/GDTCCTUploader.m index 1881a7e..81ef08a 100644 --- a/GoogleDataTransport/GDTCCTLibrary/GDTCCTUploader.m +++ b/GoogleDataTransport/GDTCCTLibrary/GDTCCTUploader.m @@ -1,5 +1,5 @@ /* - * Copyright 2019 Google + * Copyright 2020 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,67 +18,26 @@ #import "GoogleDataTransport/GDTCORLibrary/Internal/GDTCORPlatform.h" #import "GoogleDataTransport/GDTCORLibrary/Internal/GDTCORRegistrar.h" -#import "GoogleDataTransport/GDTCORLibrary/Internal/GDTCORStorageProtocol.h" #import "GoogleDataTransport/GDTCORLibrary/Public/GoogleDataTransport/GDTCORConsoleLogger.h" #import "GoogleDataTransport/GDTCORLibrary/Public/GoogleDataTransport/GDTCOREndpoints.h" #import "GoogleDataTransport/GDTCORLibrary/Public/GoogleDataTransport/GDTCOREvent.h" -#import -#import -#import - -#import "GoogleDataTransport/GDTCCTLibrary/Private/GDTCCTCompressionHelper.h" -#import "GoogleDataTransport/GDTCCTLibrary/Private/GDTCCTNanopbHelpers.h" - -#import "GoogleDataTransport/GDTCCTLibrary/Protogen/nanopb/cct.nanopb.h" +#import "GoogleDataTransport/GDTCCTLibrary/Private/GDTCCTUploadOperation.h" NS_ASSUME_NONNULL_BEGIN -#ifdef GDTCOR_VERSION -#define STR(x) STR_EXPAND(x) -#define STR_EXPAND(x) #x -static NSString *const kGDTCCTSupportSDKVersion = @STR(GDTCOR_VERSION); -#else -static NSString *const kGDTCCTSupportSDKVersion = @"UNKNOWN"; -#endif // GDTCOR_VERSION - -/** */ -static NSInteger kWeekday; - -/** */ -static NSString *const kLibraryDataCCTNextUploadTimeKey = @"GDTCCTUploaderFLLNextUploadTimeKey"; - -/** */ -static NSString *const kLibraryDataFLLNextUploadTimeKey = @"GDTCCTUploaderFLLNextUploadTimeKey"; - -#if !NDEBUG -NSNotificationName const GDTCCTUploadCompleteNotification = @"com.GDTCCTUploader.UploadComplete"; -#endif // #if !NDEBUG - -typedef void (^GDTCCTUploaderURLTaskCompletion)(NSNumber *batchID, - NSSet *_Nullable events, - NSData *_Nullable data, - NSURLResponse *_Nullable response, - NSError *_Nullable error); +@interface GDTCCTUploader () -typedef void (^GDTCCTUploaderEventBatchBlock)(NSNumber *_Nullable batchID, - NSSet *_Nullable events); +@property(nonatomic, readonly) NSOperationQueue *uploadOperationQueue; +@property(nonatomic, readonly) dispatch_queue_t uploadQueue; -@interface GDTCCTUploader () - -/// Redeclared as readwrite. -@property(nullable, nonatomic, readwrite) NSURLSessionUploadTask *currentTask; - -/// A flag indicating if there is an ongoing upload. The current implementation supports only a -/// single upload operation. If `uploadTarget` method is called when `isCurrentlyUploading == YES` -/// then no new uploads will be started. -@property(atomic) BOOL isCurrentlyUploading; +@property(nonatomic, readonly) + NSMutableDictionary *nextUploadTimeByTarget; @end @implementation GDTCCTUploader -@synthesize uploaderSession = _uploaderSession; static NSURL *_testServerURL = nil; + (void)load { @@ -98,6 +57,68 @@ + (instancetype)sharedInstance { return sharedInstance; } +- (instancetype)init { + self = [super init]; + if (self) { + _uploadQueue = dispatch_queue_create("com.google.GDTCCTUploader", DISPATCH_QUEUE_SERIAL); + _uploadOperationQueue = [[NSOperationQueue alloc] init]; + _uploadOperationQueue.maxConcurrentOperationCount = 1; + _nextUploadTimeByTarget = [[NSMutableDictionary alloc] init]; + } + return self; +} + +- (void)uploadTarget:(GDTCORTarget)target withConditions:(GDTCORUploadConditions)conditions { + // Current GDTCCTUploader expected behaviour: + // 1. Accept multiple upload request + // 2. Verify if there are events eligible for upload and start upload for the first suitable + // target + // 3. Ignore other requests while an upload is in-progress. + + // TODO: Revisit expected behaviour. + // Potentially better option: + // 1. Accept and enqueue all upload requests + // 2. Notify the client of upload stages + // 3. Allow the client cancelling upload requests as needed. + + id storage = GDTCORStoragePromiseInstanceForTarget(target); + if (storage == nil) { + GDTCORLogError(GDTCORMCEGeneralError, + @"Failed to upload target: %ld - could not find corresponding storage instance.", + (long)target); + return; + } + + GDTCCTUploadOperation *uploadOperation = + [[GDTCCTUploadOperation alloc] initWithTarget:target + conditions:conditions + uploadURL:[[self class] serverURLForTarget:target] + queue:self.uploadQueue + storage:storage + metadataProvider:self]; + + GDTCORLogDebug(@"Upload operation created: %@, target: %@", uploadOperation, @(target)); + + __weak __auto_type weakSelf = self; + __weak GDTCCTUploadOperation *weakOperation = uploadOperation; + uploadOperation.completionBlock = ^{ + GDTCORLogDebug(@"Upload operation finished: %@, uploadAttempted: %@", weakOperation, + @(weakOperation.uploadAttempted)); + + // TODO: Strongify references? + if (weakOperation.uploadAttempted) { + // Ignore all upload requests received when the upload was in progress. + [weakSelf.uploadOperationQueue cancelAllOperations]; + } + }; + + [self.uploadOperationQueue addOperation:uploadOperation]; + GDTCORLogDebug(@"Upload operation scheduled: %@, operation count: %@", uploadOperation, + @(self.uploadOperationQueue.operationCount)); +} + +#pragma mark - URLs + + (void)setTestServerURL:(NSURL *_Nullable)serverURL { _testServerURL = serverURL; } @@ -116,25 +137,7 @@ + (nullable NSURL *)serverURLForTarget:(GDTCORTarget)target { return [GDTCOREndpoints uploadURLForTarget:target]; } -- (instancetype)init { - self = [super init]; - if (self) { - _uploaderQueue = dispatch_queue_create("com.google.GDTCCTUploader", DISPATCH_QUEUE_SERIAL); - } - return self; -} - -- (NSURLSession *)uploaderSession { - if (_uploaderSession == nil) { - NSURLSessionConfiguration *config = [NSURLSessionConfiguration ephemeralSessionConfiguration]; - _uploaderSession = [NSURLSession sessionWithConfiguration:config - delegate:self - delegateQueue:nil]; - } - return _uploaderSession; -} - -- (NSString *)FLLAndCSHandINTAPIKey { +- (NSString *)FLLAndCSHAndINTAPIKey { static NSString *defaultServerKey; static dispatch_once_t onceToken; dispatch_once(&onceToken, ^{ @@ -151,511 +154,51 @@ - (NSString *)FLLAndCSHandINTAPIKey { return defaultServerKey; } -# - -- (void)uploadTarget:(GDTCORTarget)target withConditions:(GDTCORUploadConditions)conditions { - __block GDTCORBackgroundIdentifier backgroundTaskID = GDTCORBackgroundIdentifierInvalid; - - dispatch_block_t backgroundTaskCompletion = ^{ - // End the background task if there was one. - if (backgroundTaskID != GDTCORBackgroundIdentifierInvalid) { - [[GDTCORApplication sharedApplication] endBackgroundTask:backgroundTaskID]; - backgroundTaskID = GDTCORBackgroundIdentifierInvalid; - } - }; - - backgroundTaskID = [[GDTCORApplication sharedApplication] - beginBackgroundTaskWithName:@"GDTCCTUploader-upload" - expirationHandler:^{ - if (backgroundTaskID != GDTCORBackgroundIdentifierInvalid) { - // Cancel the upload and complete delivery. - [self.currentTask cancel]; - - // End the background task. - backgroundTaskCompletion(); - } - }]; - - dispatch_async(_uploaderQueue, ^{ - id storage = GDTCORStorageInstanceForTarget(target); - - // 1. Fetch events to upload. - [self batchToUploadForTarget:target - storage:storage - conditions:conditions - completion:^(NSNumber *_Nullable batchID, - NSSet *_Nullable events) { - // 2. Check if there are events to upload. - if (!events || events.count == 0) { - dispatch_async(self.uploaderQueue, ^{ - GDTCORLogDebug(@"Target %ld reported as ready for upload, but no " - @"events were selected", - (long)target); - self.isCurrentlyUploading = NO; - backgroundTaskCompletion(); - }); - - return; - } - // 3. Upload events. - [self uploadBatchWithID:batchID - events:events - target:target - storage:storage - completion:^{ - backgroundTaskCompletion(); - }]; - }]; - }); -} - -#pragma mark - Upload implementation details - -/** Performs URL request, handles the result and updates the uploader state. */ -- (void)uploadBatchWithID:(nullable NSNumber *)batchID - events:(nullable NSSet *)events - target:(GDTCORTarget)target - storage:(id)storage - completion:(dispatch_block_t)completion { - [self - sendURLRequestForBatchWithID:batchID - events:events - target:target - completionHandler:^(NSNumber *_Nonnull batchID, - NSSet *_Nullable events, NSData *_Nullable data, - NSURLResponse *_Nullable response, NSError *_Nullable error) { - dispatch_async(self.uploaderQueue, ^{ - [self handleURLResponse:response - data:data - error:error - target:target - storage:storage - batchID:batchID]; -#if !NDEBUG - // Post a notification when in DEBUG mode to state how many packages - // were uploaded. Useful for validation during tests. - [[NSNotificationCenter defaultCenter] - postNotificationName:GDTCCTUploadCompleteNotification - object:@(events.count)]; -#endif // #if !NDEBUG - self.isCurrentlyUploading = NO; - completion(); - }); - }]; -} - -/** Validates events and sends URL request and calls completion with the result. Modifies uploading - * state in the case of the failure.*/ -- (void)sendURLRequestForBatchWithID:(nullable NSNumber *)batchID - events:(nullable NSSet *)events - target:(GDTCORTarget)target - completionHandler:(GDTCCTUploaderURLTaskCompletion)completionHandler { - dispatch_async(self.uploaderQueue, ^{ - NSData *requestProtoData = [self constructRequestProtoWithEvents:events]; - NSData *gzippedData = [GDTCCTCompressionHelper gzippedData:requestProtoData]; - BOOL usingGzipData = gzippedData != nil && gzippedData.length < requestProtoData.length; - NSData *dataToSend = usingGzipData ? gzippedData : requestProtoData; - NSURLRequest *request = [self constructRequestForTarget:target data:dataToSend]; - GDTCORLogDebug(@"CTT: request containing %lu events created: %@", (unsigned long)events.count, - request); - NSSet *eventsForDebug; -#if !NDEBUG - eventsForDebug = events; -#endif - self.currentTask = [self.uploaderSession - uploadTaskWithRequest:request - fromData:dataToSend - completionHandler:^(NSData *_Nullable data, NSURLResponse *_Nullable response, - NSError *_Nullable error) { - completionHandler(batchID, eventsForDebug, data, response, error); - }]; - GDTCORLogDebug(@"%@", @"CCT: The upload task is about to begin."); - [self.currentTask resume]; - }); -} - -/** Handles URL request response. */ -- (void)handleURLResponse:(nullable NSURLResponse *)response - data:(nullable NSData *)data - error:(nullable NSError *)error - target:(GDTCORTarget)target - storage:(id)storage - batchID:(NSNumber *)batchID { - GDTCORLogDebug(@"%@", @"CCT: request completed"); - if (error) { - GDTCORLogWarning(GDTCORMCWUploadFailed, @"There was an error uploading events: %@", error); - } - NSError *decodingError; - GDTCORClock *futureUploadTime; - if (data) { - gdt_cct_LogResponse logResponse = GDTCCTDecodeLogResponse(data, &decodingError); - if (!decodingError && logResponse.has_next_request_wait_millis) { - GDTCORLogDebug(@"CCT: The backend responded asking to not upload for %lld millis from now.", - logResponse.next_request_wait_millis); - futureUploadTime = - [GDTCORClock clockSnapshotInTheFuture:logResponse.next_request_wait_millis]; - } else if (decodingError) { - GDTCORLogDebug(@"There was a response decoding error: %@", decodingError); - } - pb_release(gdt_cct_LogResponse_fields, &logResponse); - } - if (!futureUploadTime) { - GDTCORLogDebug(@"%@", @"CCT: The backend response failed to parse, so the next request " - @"won't occur until 15 minutes from now"); - // 15 minutes from now. - futureUploadTime = [GDTCORClock clockSnapshotInTheFuture:15 * 60 * 1000]; - } - switch (target) { - case kGDTCORTargetCCT: - self->_CCTNextUploadTime = futureUploadTime; - break; - - case kGDTCORTargetFLL: - // Falls through. - case kGDTCORTargetINT: - // Falls through. - case kGDTCORTargetCSH: - self->_FLLNextUploadTime = futureUploadTime; - break; - default: - break; - } +#pragma mark - GDTCCTUploadMetadataProvider - // Only retry if one of these codes is returned, or there was an error. - if (error || ((NSHTTPURLResponse *)response).statusCode == 429 || - ((NSHTTPURLResponse *)response).statusCode == 503) { - // Move the events back to the main storage to be uploaded on the next attempt. - [storage removeBatchWithID:batchID deleteEvents:NO onComplete:nil]; - } else { - GDTCORLogDebug(@"%@", @"CCT: package delivered"); - [storage removeBatchWithID:batchID deleteEvents:YES onComplete:nil]; +- (nullable GDTCORClock *)nextUploadTimeForTarget:(GDTCORTarget)target { + @synchronized(self.nextUploadTimeByTarget) { + return self.nextUploadTimeByTarget[@(target)]; } - - self.currentTask = nil; } -#pragma mark - Stored events upload - -/** Fetches a batch of pending events for the specified target and conditions. Passes `nil` to - * completion if there are no suitable events to upload. */ -- (void)batchToUploadForTarget:(GDTCORTarget)target - storage:(id)storage - conditions:(GDTCORUploadConditions)conditions - completion:(GDTCCTUploaderEventBatchBlock)completion { - // 1. Check if the conditions for the target are suitable. - if (![self readyToUploadTarget:target conditions:conditions]) { - completion(nil, nil); - return; +- (void)setNextUploadTime:(nullable GDTCORClock *)time forTarget:(GDTCORTarget)target { + @synchronized(self.nextUploadTimeByTarget) { + self.nextUploadTimeByTarget[@(target)] = time; } - - // 2. Remove previously attempted batches - [self removeBatchesForTarget:target - storage:storage - onComplete:^{ - // There may be a big amount of events stored, so creating a batch may be an - // expensive operation. - - // 3. Do a lightweight check if there are any events for the target first to - // finish early if there are no. - [storage hasEventsForTarget:target - onComplete:^(BOOL hasEvents) { - // 4. Proceed with fetching the events. - [self batchToUploadForTarget:target - storage:storage - conditions:conditions - hasEvents:hasEvents - completion:completion]; - }]; - }]; -} - -/** Makes final checks before and makes */ -- (void)batchToUploadForTarget:(GDTCORTarget)target - storage:(id)storage - conditions:(GDTCORUploadConditions)conditions - hasEvents:(BOOL)hasEvents - completion:(GDTCCTUploaderEventBatchBlock)completion { - dispatch_async(self.uploaderQueue, ^{ - if (!hasEvents) { - // No events to upload. - completion(nil, nil); - return; - } - - // Check if the conditions are still met before starting upload. - if (![self readyToUploadTarget:target conditions:conditions]) { - completion(nil, nil); - return; - } - - // All conditions have been checked and met. Lock uploader for this target to prevent other - // targets upload attempts. - self.isCurrentlyUploading = YES; - - // Fetch a batch to upload and pass along. - GDTCORStorageEventSelector *eventSelector = [self eventSelectorTarget:target - withConditions:conditions]; - [storage batchWithEventSelector:eventSelector - batchExpiration:[NSDate dateWithTimeIntervalSinceNow:600] - onComplete:completion]; - }); -} - -- (void)removeBatchesForTarget:(GDTCORTarget)target - storage:(id)storage - onComplete:(dispatch_block_t)onComplete { - [storage batchIDsForTarget:target - onComplete:^(NSSet *_Nullable batchIDs) { - // No stored batches, no need to remove anything. - if (batchIDs.count < 1) { - onComplete(); - return; - } - - dispatch_group_t dispatchGroup = dispatch_group_create(); - for (NSNumber *batchID in batchIDs) { - dispatch_group_enter(dispatchGroup); - - // Remove batches and moves events back to the storage. - [storage removeBatchWithID:batchID - deleteEvents:NO - onComplete:^{ - dispatch_group_leave(dispatchGroup); - }]; - } - - // Wait until all batches are removed and call completion handler. - dispatch_group_notify(dispatchGroup, self.uploaderQueue, ^{ - onComplete(); - }); - }]; } -#pragma mark - Private helper methods - -/** */ -- (BOOL)readyToUploadTarget:(GDTCORTarget)target conditions:(GDTCORUploadConditions)conditions { - if (self.isCurrentlyUploading) { - GDTCORLogDebug(@"%@", @"CCT: Wait until previous upload finishes. The current version supports " - @"only a single batch uploading at the time."); - return NO; - } - - // Not ready to upload with no network connection. - // TODO: Reconsider using reachability to prevent an upload attempt. - // See https://developer.apple.com/videos/play/wwdc2019/712/ (49:40) for more details. - if (conditions & GDTCORUploadConditionNoNetwork) { - GDTCORLogDebug(@"%@", @"CCT: Not ready to upload without a network connection."); - return NO; - } - - // Upload events when there are with no additional conditions for kGDTCORTargetCSH. - if (target == kGDTCORTargetCSH) { - GDTCORLogDebug(@"%@", @"CCT: kGDTCORTargetCSH events are allowed to be " - @"uploaded straight away."); - return YES; - } - - if (target == kGDTCORTargetINT) { - GDTCORLogDebug(@"%@", @"CCT: kGDTCORTargetINT events are allowed to be " - @"uploaded straight away."); - return YES; - } - - // Upload events with no additional conditions if high priority. - if ((conditions & GDTCORUploadConditionHighPriority) == GDTCORUploadConditionHighPriority) { - GDTCORLogDebug(@"%@", @"CCT: a high priority event is allowing an upload"); - return YES; - } - - // Check next upload time for the target. - BOOL isAfterNextUploadTime = YES; - switch (target) { - case kGDTCORTargetCCT: - if (self->_CCTNextUploadTime) { - isAfterNextUploadTime = [[GDTCORClock snapshot] isAfter:self->_CCTNextUploadTime]; - } - break; - - case kGDTCORTargetFLL: - if (self->_FLLNextUploadTime) { - isAfterNextUploadTime = [[GDTCORClock snapshot] isAfter:self->_FLLNextUploadTime]; - } - break; - - default: - // The CSH backend should be handled above. - break; - } - - if (isAfterNextUploadTime) { - GDTCORLogDebug(@"CCT: can upload to target %ld because the request wait time has transpired", - (long)target); - } else { - GDTCORLogDebug(@"CCT: can't upload to target %ld because the backend asked to wait", - (long)target); - } - - return isAfterNextUploadTime; -} - -/** Constructs data given an upload package. - * - * @param events The events used to construct the request proto bytes. - * @return Proto bytes representing a gdt_cct_LogRequest object. - */ -- (nonnull NSData *)constructRequestProtoWithEvents:(NSSet *)events { - // Segment the log events by log type. - NSMutableDictionary *> *logMappingIDToLogSet = - [[NSMutableDictionary alloc] init]; - [events enumerateObjectsUsingBlock:^(GDTCOREvent *_Nonnull event, BOOL *_Nonnull stop) { - NSMutableSet *logSet = logMappingIDToLogSet[event.mappingID]; - logSet = logSet ? logSet : [[NSMutableSet alloc] init]; - [logSet addObject:event]; - logMappingIDToLogSet[event.mappingID] = logSet; - }]; - - gdt_cct_BatchedLogRequest batchedLogRequest = - GDTCCTConstructBatchedLogRequest(logMappingIDToLogSet); - - NSData *data = GDTCCTEncodeBatchedLogRequest(&batchedLogRequest); - pb_release(gdt_cct_BatchedLogRequest_fields, &batchedLogRequest); - return data ? data : [[NSData alloc] init]; -} - -/** Constructs a request to FLL given a URL and request body data. - * - * @param target The target backend to send the request to. - * @param data The request body data. - * @return A new NSURLRequest ready to be sent to FLL. - */ -- (nullable NSURLRequest *)constructRequestForTarget:(GDTCORTarget)target data:(NSData *)data { - if (data == nil || data.length == 0) { - GDTCORLogDebug(@"There was no data to construct a request for target %ld.", (long)target); - return nil; - } - NSURL *URL = [[self class] serverURLForTarget:target]; - NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:URL]; - NSString *targetString; - switch (target) { - case kGDTCORTargetCCT: - targetString = @"cct"; - break; - - case kGDTCORTargetFLL: - targetString = @"fll"; - break; - - case kGDTCORTargetCSH: - targetString = @"csh"; - break; - case kGDTCORTargetINT: - targetString = @"int"; - break; - - default: - targetString = @"unknown"; - break; - } - NSString *userAgent = - [NSString stringWithFormat:@"datatransport/%@ %@support/%@ apple/", kGDTCORVersion, - targetString, kGDTCCTSupportSDKVersion]; +- (nullable NSString *)APIKeyForTarget:(GDTCORTarget)target { if (target == kGDTCORTargetFLL || target == kGDTCORTargetCSH) { - [request setValue:[self FLLAndCSHandINTAPIKey] forHTTPHeaderField:@"X-Goog-Api-Key"]; + return [self FLLAndCSHAndINTAPIKey]; } if (target == kGDTCORTargetINT) { - [request setValue:[self FLLAndCSHandINTAPIKey] forHTTPHeaderField:@"X-Goog-Api-Key"]; - } - - if ([GDTCCTCompressionHelper isGzipped:data]) { - [request setValue:@"gzip" forHTTPHeaderField:@"Content-Encoding"]; - } - [request setValue:@"application/x-protobuf" forHTTPHeaderField:@"Content-Type"]; - [request setValue:@"gzip" forHTTPHeaderField:@"Accept-Encoding"]; - [request setValue:userAgent forHTTPHeaderField:@"User-Agent"]; - request.HTTPMethod = @"POST"; - [request setHTTPBody:data]; - return request; -} - -/** */ -- (nullable GDTCORStorageEventSelector *)eventSelectorTarget:(GDTCORTarget)target - withConditions:(GDTCORUploadConditions)conditions { - id storage = GDTCORStorageInstanceForTarget(target); - if ((conditions & GDTCORUploadConditionHighPriority) == GDTCORUploadConditionHighPriority) { - return [GDTCORStorageEventSelector eventSelectorForTarget:target]; + return [self FLLAndCSHAndINTAPIKey]; } - NSMutableSet *qosTiers = [[NSMutableSet alloc] init]; - if (conditions & GDTCORUploadConditionWifiData) { - [qosTiers addObjectsFromArray:@[ - @(GDTCOREventQoSFast), @(GDTCOREventQoSWifiOnly), @(GDTCOREventQosDefault), - @(GDTCOREventQoSTelemetry), @(GDTCOREventQoSUnknown) - ]]; - } - if (conditions & GDTCORUploadConditionMobileData) { - [qosTiers addObjectsFromArray:@[ @(GDTCOREventQoSFast), @(GDTCOREventQosDefault) ]]; - } - - __block NSInteger lastDayOfDailyUpload; - NSString *lastDailyUploadDataKey = [NSString - stringWithFormat:@"%@LastDailyUpload-%ld", NSStringFromClass([self class]), (long)target]; - [storage libraryDataForKey:lastDailyUploadDataKey - onFetchComplete:^(NSData *_Nullable data, NSError *_Nullable error) { - [data getBytes:&lastDayOfDailyUpload length:sizeof(NSInteger)]; - } - setNewValue:^NSData *_Nullable { - if (lastDayOfDailyUpload != kWeekday) { - return [NSData dataWithBytes:&lastDayOfDailyUpload length:sizeof(NSInteger)]; - } - return nil; - }]; - - return [[GDTCORStorageEventSelector alloc] initWithTarget:target - eventIDs:nil - mappingIDs:nil - qosTiers:qosTiers]; -} -#pragma mark - GDTCORLifecycleProtocol - -- (void)appWillForeground:(GDTCORApplication *)app { - dispatch_async(_uploaderQueue, ^{ - NSDateComponents *dateComponents = [[NSDateComponents alloc] init]; - NSCalendar *gregorianCalendar = - [[NSCalendar alloc] initWithCalendarIdentifier:NSCalendarIdentifierGregorian]; - NSDate *date = [gregorianCalendar dateFromComponents:dateComponents]; - kWeekday = [gregorianCalendar component:NSCalendarUnitWeekday fromDate:date]; - }); + return nil; } -- (void)appWillTerminate:(GDTCORApplication *)application { - dispatch_sync(_uploaderQueue, ^{ - [self.currentTask cancel]; - }); -} - -#pragma mark - NSURLSessionDelegate +#if !NDEBUG -- (void)URLSession:(NSURLSession *)session - task:(NSURLSessionTask *)task - willPerformHTTPRedirection:(NSHTTPURLResponse *)response - newRequest:(NSURLRequest *)request - completionHandler:(void (^)(NSURLRequest *_Nullable))completionHandler { - if (!completionHandler) { - return; - } - if (response.statusCode == 302 || response.statusCode == 301) { - if ([request.URL isEqual:[[self class] serverURLForTarget:kGDTCORTargetFLL]]) { - NSURLRequest *newRequest = [self constructRequestForTarget:kGDTCORTargetCCT - data:task.originalRequest.HTTPBody]; - completionHandler(newRequest); +- (BOOL)waitForUploadFinishedWithTimeout:(NSTimeInterval)timeout { + NSDate *expirationDate = [NSDate dateWithTimeIntervalSinceNow:timeout]; + while ([expirationDate compare:[NSDate date]] == NSOrderedDescending) { + if (self.uploadOperationQueue.operationCount == 0) { + return YES; + } else { + [[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.1]]; } - } else { - completionHandler(request); } + + GDTCORLogDebug(@"Uploader wait for finish timeout exceeded. Operations still in queue: %@", + self.uploadOperationQueue.operations); + return NO; } +#endif // !NDEBUG + @end NS_ASSUME_NONNULL_END diff --git a/GoogleDataTransport/GDTCCTLibrary/Private/GDTCCTUploadOperation.h b/GoogleDataTransport/GDTCCTLibrary/Private/GDTCCTUploadOperation.h new file mode 100644 index 0000000..ad5b044 --- /dev/null +++ b/GoogleDataTransport/GDTCCTLibrary/Private/GDTCCTUploadOperation.h @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import "GoogleDataTransport/GDTCORLibrary/Internal/GDTCORUploader.h" + +@protocol GDTCORStoragePromiseProtocol; + +NS_ASSUME_NONNULL_BEGIN + +// TODO: Refine API and API docs + +@protocol GDTCCTUploadMetadataProvider + +- (nullable GDTCORClock *)nextUploadTimeForTarget:(GDTCORTarget)target; +- (void)setNextUploadTime:(nullable GDTCORClock *)time forTarget:(GDTCORTarget)target; + +- (nullable NSString *)APIKeyForTarget:(GDTCORTarget)target; + +@end + +/** Class capable of uploading events to the CCT backend. */ +@interface GDTCCTUploadOperation : NSOperation + +- (instancetype)init NS_UNAVAILABLE; + +- (instancetype)initWithTarget:(GDTCORTarget)target + conditions:(GDTCORUploadConditions)conditions + uploadURL:(NSURL *)uploadURL + queue:(dispatch_queue_t)queue + storage:(id)storage + metadataProvider:(id)metadataProvider; + +/** YES if a batch upload attempt was performed. NO otherwise. If NO for the finished operation, + * then there were no events suitable for upload. */ +@property(nonatomic, readonly) BOOL uploadAttempted; + +/** The queue on which all CCT uploading will occur. */ +@property(nonatomic, readonly) dispatch_queue_t uploaderQueue; + +/** The URL session that will attempt upload. */ +@property(nonatomic, readonly) NSURLSession *uploaderSession; + +/** The current upload task. */ +@property(nullable, nonatomic, readonly) NSURLSessionUploadTask *currentTask; + +@end + +NS_ASSUME_NONNULL_END diff --git a/GoogleDataTransport/GDTCCTLibrary/Private/GDTCCTUploader.h b/GoogleDataTransport/GDTCCTLibrary/Private/GDTCCTUploader.h index 1bdffd5..876fbe1 100644 --- a/GoogleDataTransport/GDTCCTLibrary/Private/GDTCCTUploader.h +++ b/GoogleDataTransport/GDTCCTLibrary/Private/GDTCCTUploader.h @@ -20,40 +20,25 @@ NS_ASSUME_NONNULL_BEGIN -#if !NDEBUG -/** A notification fired when uploading is complete, detailing the number of events uploaded. */ -extern NSNotificationName const GDTCCTUploadCompleteNotification; -#endif // #if !NDEBUG - /** Class capable of uploading events to the CCT backend. */ @interface GDTCCTUploader : NSObject -/** The queue on which all CCT uploading will occur. */ -@property(nonatomic, readonly) dispatch_queue_t uploaderQueue; - -/** The URL session that will attempt upload. */ -@property(nonatomic, readonly) NSURLSession *uploaderSession; - -/** The current upload task. */ -@property(nullable, nonatomic, readonly) NSURLSessionUploadTask *currentTask; - -/** The next upload time for the CCT target. */ -@property(nullable, nonatomic) GDTCORClock *CCTNextUploadTime; - -/** The next upload time for the FLL target. */ -@property(nullable, nonatomic) GDTCORClock *FLLNextUploadTime; +/** Creates and/or returns the singleton instance of this class. + * + * @return The singleton instance of this class. + */ ++ (instancetype)sharedInstance; #if !NDEBUG /** An upload URL used across all targets. For testing only. */ @property(class, nullable, nonatomic) NSURL *testServerURL; -#endif // !NDEBUG - -/** Creates and/or returns the singleton instance of this class. - * - * @return The singleton instance of this class. +/** Spins runloop until upload finishes or timeout. + * @return YES if upload finishes, NO in the case of timeout. */ -+ (instancetype)sharedInstance; +- (BOOL)waitForUploadFinishedWithTimeout:(NSTimeInterval)timeout; + +#endif // !NDEBUG @end diff --git a/GoogleDataTransport/GDTCCTTests/Common/TestStorage/GDTCCTTestStorage.h b/GoogleDataTransport/GDTCCTTests/Common/TestStorage/GDTCCTTestStorage.h index 20828f2..9fd7751 100644 --- a/GoogleDataTransport/GDTCCTTests/Common/TestStorage/GDTCCTTestStorage.h +++ b/GoogleDataTransport/GDTCCTTests/Common/TestStorage/GDTCCTTestStorage.h @@ -29,7 +29,8 @@ typedef void (^GDTCCTTestStorageHasEventsCompletion)(BOOL hasEvents); typedef void (^GDTCCTTestStorageHasEventsHandler)(GDTCORTarget target, GDTCCTTestStorageHasEventsCompletion completion); -@interface GDTCCTTestStorage : NSObject +// TODO: Add GDTCORStoragePromiseProtocol support to fix tests. +@interface GDTCCTTestStorage : NSObject #pragma mark - Method call expectations. diff --git a/GoogleDataTransport/GDTCCTTests/Common/TestStorage/GDTCCTTestStorage.m b/GoogleDataTransport/GDTCCTTests/Common/TestStorage/GDTCCTTestStorage.m index 3231a10..aa7bc0d 100644 --- a/GoogleDataTransport/GDTCCTTests/Common/TestStorage/GDTCCTTestStorage.m +++ b/GoogleDataTransport/GDTCCTTests/Common/TestStorage/GDTCCTTestStorage.m @@ -16,8 +16,15 @@ #import "GoogleDataTransport/GDTCCTTests/Common/TestStorage/GDTCCTTestStorage.h" +#import "GoogleDataTransport/GDTCORLibrary/Private/GDTCORUploadBatch.h" #import "GoogleDataTransport/GDTCORLibrary/Public/GoogleDataTransport/GDTCOREvent.h" +#if __has_include() +#import +#else +#import "FBLPromises.h" +#endif + @implementation GDTCCTTestStorage { /** Store the events in memory. */ NSMutableDictionary *_storedEvents; @@ -141,4 +148,69 @@ - (void)defaultBatchWithEventSelector:(nonnull GDTCORStorageEventSelector *)even } } +- (FBLPromise *> *)batchIDsForTarget:(GDTCORTarget)target { + return [FBLPromise wrapObjectCompletion:^(FBLPromiseObjectCompletion _Nonnull handler) { + [self batchIDsForTarget:target onComplete:handler]; + }]; +} + +- (FBLPromise *)removeBatchWithID:(NSNumber *)batchID deleteEvents:(BOOL)deleteEvents { + return [FBLPromise wrapCompletion:^(FBLPromiseCompletion _Nonnull handler) { + [self removeBatchWithID:batchID deleteEvents:deleteEvents onComplete:handler]; + }]; +} + +- (FBLPromise *)removeBatchesWithIDs:(NSSet *)batchIDs + deleteEvents:(BOOL)deleteEvents { + NSMutableArray *removeBatchPromises = + [NSMutableArray arrayWithCapacity:batchIDs.count]; + for (NSNumber *batchID in batchIDs) { + [removeBatchPromises addObject:[self removeBatchWithID:batchID deleteEvents:deleteEvents]]; + } + + return [FBLPromise all:[removeBatchPromises copy]].then(^id(id result) { + return [FBLPromise resolvedWith:[NSNull null]]; + }); +} + +- (FBLPromise *)removeAllBatchesForTarget:(GDTCORTarget)target + deleteEvents:(BOOL)deleteEvents { + return [self batchIDsForTarget:target].then(^id(NSSet *batchIDs) { + return [self removeBatchesWithIDs:batchIDs deleteEvents:NO]; + }); +} + +- (FBLPromise *)hasEventsForTarget:(GDTCORTarget)target { + return [FBLPromise wrapBoolCompletion:^(FBLPromiseBoolCompletion _Nonnull handler) { + [self hasEventsForTarget:target onComplete:handler]; + }]; +} + +- (FBLPromise *)batchWithEventSelector: + (GDTCORStorageEventSelector *)eventSelector + batchExpiration:(NSDate *)expiration { + return [FBLPromise + async:^(FBLPromiseFulfillBlock _Nonnull fulfill, FBLPromiseRejectBlock _Nonnull reject) { + [self batchWithEventSelector:eventSelector + batchExpiration:expiration + onComplete:^(NSNumber *_Nullable newBatchID, + NSSet *_Nullable batchEvents) { + if (newBatchID == nil || batchEvents == nil) { + reject([self genericRejectedPromiseErrorWithReason: + @"There are no events for the selector."]); + } else { + fulfill([[GDTCORUploadBatch alloc] initWithBatchID:newBatchID + events:batchEvents]); + } + }]; + }]; +} + +// TODO: More comprehensive error. +- (NSError *)genericRejectedPromiseErrorWithReason:(NSString *)reason { + return [NSError errorWithDomain:@"GDTCORFlatFileStorage" + code:-1 + userInfo:@{NSLocalizedFailureReasonErrorKey : reason}]; +} + @end diff --git a/GoogleDataTransport/GDTCCTTests/Integration/GDTCCTIntegrationTest.m b/GoogleDataTransport/GDTCCTTests/Integration/GDTCCTIntegrationTest.m index d637ab6..55af646 100644 --- a/GoogleDataTransport/GDTCCTTests/Integration/GDTCCTIntegrationTest.m +++ b/GoogleDataTransport/GDTCCTTests/Integration/GDTCCTIntegrationTest.m @@ -16,6 +16,7 @@ #import +#import "GoogleDataTransport/GDTCORLibrary/Public/GoogleDataTransport/GDTCORConsoleLogger.h" #import "GoogleDataTransport/GDTCORLibrary/Public/GoogleDataTransport/GDTCOREvent.h" #import "GoogleDataTransport/GDTCORLibrary/Public/GoogleDataTransport/GDTCOREventDataObject.h" #import "GoogleDataTransport/GDTCORLibrary/Public/GoogleDataTransport/GDTCORTransport.h" @@ -23,115 +24,81 @@ #import #import "GoogleDataTransport/GDTCCTLibrary/Private/GDTCCTUploader.h" +#import "GoogleDataTransport/GDTCCTTests/Unit/TestServer/GDTCCTTestServer.h" +#import "GoogleDataTransport/GDTCORTests/Common/Categories/GDTCORFlatFileStorage+Testing.h" -typedef void (^GDTCCTIntegrationTestBlock)(NSURLSessionUploadTask *_Nullable); - -@interface GDTCCTTestDataObject : NSObject +#import +#import +#import +#import "GoogleDataTransport/GDTCCTLibrary/Protogen/nanopb/cct.nanopb.h" +@interface NSData (GDTCOREventDataObject) @end -@implementation GDTCCTTestDataObject +@implementation NSData (GDTCOREventDataObject) - (NSData *)transportBytes { - // Return some random event data corresponding to mapping ID 1018. - NSBundle *testBundle = [NSBundle bundleForClass:[self class]]; - NSArray *dataFiles = @[ - @"message-32347456.dat", @"message-35458880.dat", @"message-39882816.dat", - @"message-40043840.dat", @"message-40657984.dat" - ]; - NSURL *fileURL = [testBundle URLForResource:dataFiles[arc4random_uniform(5)] withExtension:nil]; - return [NSData dataWithContentsOfURL:fileURL]; + return [self copy]; } @end @interface GDTCCTIntegrationTest : XCTestCase - -/** If YES, the network conditions were good enough to allow running integration tests. */ -@property(nonatomic) BOOL okToRunTest; - /** If YES, allow the recursive generating of events. */ @property(nonatomic) BOOL generateEvents; /** The total number of events generated for this test. */ @property(nonatomic) NSInteger totalEventsGenerated; +/** Events passed to GDT to be sent. */ +@property(nonatomic) NSMutableArray *scheduledEvents; + +/** Events decoded from the request received by the server. */ +@property(nonatomic) NSMutableArray *serverReceivedEvents; + /** The transporter used by the test. */ @property(nonatomic) GDTCORTransport *transport; -/** The local notification listener, to be removed after each test. */ -@property(nonatomic, strong) id uploadObserver; +/** The local HTTP server to use for testing. */ +@property(nonatomic) GDTCCTTestServer *testServer; @end @implementation GDTCCTIntegrationTest - (void)setUp { + // Make sure clean storage state before start. + [[GDTCORFlatFileStorage sharedInstance] reset]; + + // Reset events. + self.scheduledEvents = [NSMutableArray array]; + self.serverReceivedEvents = [NSMutableArray array]; + // Don't recursively generate events by default. self.generateEvents = NO; self.totalEventsGenerated = 0; - dispatch_semaphore_t sema = dispatch_semaphore_create(0); - NSURLSession *session = [NSURLSession sharedSession]; - NSURLSessionDataTask *task = - [session dataTaskWithURL:[NSURL URLWithString:@"https://google.com"] - completionHandler:^(NSData *_Nullable data, NSURLResponse *_Nullable response, - NSError *_Nullable error) { - if (error) { - self.okToRunTest = NO; - } else { - self.okToRunTest = YES; - } - self.transport = [[GDTCORTransport alloc] initWithMappingID:@"1018" - transformers:nil - target:kGDTCORTargetCSH]; - dispatch_semaphore_signal(sema); - }]; - [task resume]; - dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, 10.0 * NSEC_PER_SEC)); -} -- (void)tearDown { - if (self.uploadObserver) { - [[NSNotificationCenter defaultCenter] removeObserver:self.uploadObserver]; - self.uploadObserver = nil; - } + self.testServer = [[GDTCCTTestServer alloc] init]; + self.testServer.responseNextRequestWaitTime = 0; + [self.testServer registerLogBatchPath]; + [self.testServer start]; + XCTAssertTrue(self.testServer.isRunning); - [super tearDown]; -} + GDTCCTUploader.testServerURL = + [self.testServer.serverURL URLByAppendingPathComponent:@"logBatch"]; -/** Generates an event and sends it through the transport infrastructure. */ -- (void)generateEventWithQoSTier:(GDTCOREventQoS)qosTier { - GDTCOREvent *event = [self.transport eventForTransport]; - event.dataObject = [[GDTCCTTestDataObject alloc] init]; - event.qosTier = qosTier; - [self.transport sendDataEvent:event - onComplete:^(BOOL wasWritten, NSError *_Nullable error) { - NSLog(@"Storing a data event completed."); - }]; - dispatch_async(dispatch_get_main_queue(), ^{ - self.totalEventsGenerated += 1; - }); + self.transport = [[GDTCORTransport alloc] initWithMappingID:@"1018" + transformers:nil + target:kGDTCORTargetCSH]; } -/** Generates events recursively at random intervals between 0 and 5 seconds. */ -- (void)recursivelyGenerateEvent { - if (self.generateEvents) { - [self generateEventWithQoSTier:GDTCOREventQosDefault]; - dispatch_after( - dispatch_time(DISPATCH_TIME_NOW, (int64_t)(arc4random_uniform(6) * NSEC_PER_SEC)), - dispatch_get_main_queue(), ^{ - [self recursivelyGenerateEvent]; - }); - } +- (void)tearDown { + XCTAssert([[GDTCCTUploader sharedInstance] waitForUploadFinishedWithTimeout:2]); + [super tearDown]; } /** Tests sending data to CCT with a high priority event if network conditions are good. */ - (void)testSendingDataToCCT { - if (!self.okToRunTest) { - NSLog(@"Skipping the integration test, as the network conditions weren't good enough."); - return; - } - // Send a number of events across multiple queues in order to ensure the threading is working as // expected. dispatch_queue_t queue1 = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); @@ -151,32 +118,25 @@ - (void)testSendingDataToCCT { } } - XCTestExpectation *eventsUploaded = - [self expectationWithDescription:@"Events were successfully uploaded to CCT."]; - [eventsUploaded setAssertForOverFulfill:NO]; - self.uploadObserver = [self uploadNotificationObserverWithExpectation:eventsUploaded]; + XCTestExpectation *eventsUploaded = [self expectationForEventsToUpload]; // Send a high priority event to flush events. [self generateEventWithQoSTier:GDTCOREventQoSFast]; - // Validate that at least one event was uploaded. + // Wait for events to be uploaded. [self waitForExpectations:@[ eventsUploaded ] timeout:60.0]; + + // Validate sent events content. + [self assertAllScheduledEventsWereReceived]; } - (void)testRunsWithoutCrashing { - if (!self.okToRunTest) { - NSLog(@"Skipping the integration test, as the network conditions weren't good enough."); - return; - } // Just run for a minute whilst generating events. NSInteger secondsToRun = 65; self.generateEvents = YES; - XCTestExpectation *eventsUploaded = - [self expectationWithDescription:@"Events were successfully uploaded to CCT."]; - [eventsUploaded setAssertForOverFulfill:NO]; - - self.uploadObserver = [self uploadNotificationObserverWithExpectation:eventsUploaded]; + XCTestExpectation *eventsUploaded = [self expectationForEventsToUpload]; + eventsUploaded.expectedFulfillmentCount = 2; [self recursivelyGenerateEvent]; @@ -186,32 +146,148 @@ - (void)testRunsWithoutCrashing { // Send a high priority event to flush other events. [self generateEventWithQoSTier:GDTCOREventQoSFast]; - - [self waitForExpectations:@[ eventsUploaded ] timeout:60.0]; }); - [[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:secondsToRun + 5]]; + + // Wait for events to be uploaded. + [self waitForExpectations:@[ eventsUploaded ] timeout:secondsToRun + 5]; + + // Validate sent events content. + [self assertAllScheduledEventsWereReceived]; +} + +#pragma mark - Helpers + +/** Generates an event and sends it through the transport infrastructure. */ +- (void)generateEventWithQoSTier:(GDTCOREventQoS)qosTier { + GDTCOREvent *event = [self.transport eventForTransport]; + event.dataObject = [[[NSUUID UUID] UUIDString] dataUsingEncoding:NSUTF8StringEncoding]; + event.qosTier = qosTier; + [self.transport sendDataEvent:event + onComplete:^(BOOL wasWritten, NSError *_Nullable error) { + NSLog(@"Storing a data event completed."); + }]; + dispatch_async(dispatch_get_main_queue(), ^{ + self.totalEventsGenerated += 1; + [self.scheduledEvents addObject:event]; + }); } -/** Registers a notification observer for when an upload occurs and returns the observer. */ -- (id)uploadNotificationObserverWithExpectation:(XCTestExpectation *)expectation { - return [[NSNotificationCenter defaultCenter] - addObserverForName:GDTCCTUploadCompleteNotification - object:nil - queue:nil - usingBlock:^(NSNotification *_Nonnull note) { - NSNumber *eventsUploadedNumber = note.object; - if (![eventsUploadedNumber isKindOfClass:[NSNumber class]]) { - XCTFail(@"Expected notification object of events uploaded, " - @"instead got a %@.", - [eventsUploadedNumber class]); - } - // We don't necessarily need *all* uploads to have happened, just some (due to - // timing). As long as there are some events uploaded, call it a success. - NSInteger eventsUploaded = eventsUploadedNumber.integerValue; - if (eventsUploaded > 0 && eventsUploaded <= self.totalEventsGenerated) { - [expectation fulfill]; - } - }]; +/** Generates events recursively at random intervals between 0 and 5 seconds. */ +- (void)recursivelyGenerateEvent { + if (self.generateEvents) { + [self generateEventWithQoSTier:GDTCOREventQosDefault]; + dispatch_after( + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(arc4random_uniform(6) * NSEC_PER_SEC)), + dispatch_get_main_queue(), ^{ + [self recursivelyGenerateEvent]; + }); + } +} + +- (XCTestExpectation *)expectationForEventsToUpload { + XCTestExpectation *responseSentExpectation = [self expectationWithDescription:@"response sent"]; + + __auto_type __weak weakSelf = self; + self.testServer.requestHandler = + ^(GCDWebServerRequest *_Nonnull request, GCDWebServerResponse *_Nullable suggestedResponse, + GCDWebServerCompletionBlock _Nonnull completionBlock) { + __auto_type strongSelf = weakSelf; + if (strongSelf == nil) { + XCTFail(); + return; + } + + // Decode events. + GCDWebServerDataRequest *dataRequest = (GCDWebServerDataRequest *)request; + NSError *decodeError; + gdt_cct_BatchedLogRequest decodedRequest = [weakSelf requestWithData:dataRequest.data + error:&decodeError]; + __auto_type events = [weakSelf eventsWithBatchRequest:decodedRequest error:&decodeError]; + XCTAssertNil(decodeError); + [weakSelf.serverReceivedEvents addObjectsFromArray:events]; + + // Send response. + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.3 * NSEC_PER_SEC)), + dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0), ^{ + completionBlock(suggestedResponse); + }); + + [responseSentExpectation fulfill]; + }; + + return responseSentExpectation; +} + +- (gdt_cct_BatchedLogRequest)requestWithData:(NSData *)data error:(NSError **)outError { + gdt_cct_BatchedLogRequest request = gdt_cct_BatchedLogRequest_init_default; + pb_istream_t istream = pb_istream_from_buffer([data bytes], [data length]); + if (!pb_decode(&istream, gdt_cct_BatchedLogRequest_fields, &request)) { + NSString *nanopb_error = [NSString stringWithFormat:@"%s", PB_GET_ERROR(&istream)]; + NSDictionary *userInfo = @{@"nanopb error:" : nanopb_error}; + if (outError != NULL) { + *outError = [NSError errorWithDomain:NSURLErrorDomain code:-1 userInfo:userInfo]; + } + request = (gdt_cct_BatchedLogRequest)gdt_cct_BatchedLogRequest_init_default; + } + return request; +} + +- (NSArray *)eventsWithBatchRequest:(gdt_cct_BatchedLogRequest)batchRequest + error:(NSError **)outError { + NSMutableArray *events = [NSMutableArray array]; + + for (NSUInteger reqIdx = 0; reqIdx < batchRequest.log_request_count; reqIdx++) { + gdt_cct_LogRequest request = batchRequest.log_request[reqIdx]; + + NSString *mappingID = @(request.log_source).stringValue; + + for (NSUInteger eventIdx = 0; eventIdx < request.log_event_count; eventIdx++) { + gdt_cct_LogEvent event = request.log_event[eventIdx]; + + GDTCOREvent *decodedEvent = [[GDTCOREvent alloc] initWithMappingID:mappingID + target:kGDTCORTargetTest]; + decodedEvent.dataObject = [NSData dataWithBytes:event.source_extension->bytes + length:event.source_extension->size]; + + [events addObject:decodedEvent]; + } + } + + return [events copy]; +} + +- (void)assertAllScheduledEventsWereReceived { + // Assume unique payload. + __auto_type scheduledEventsByPayload = + [self eventsByPayloadWithEvents:[self.scheduledEvents copy]]; + __auto_type receivedEventsByPayload = + [self eventsByPayloadWithEvents:[self.serverReceivedEvents copy]]; + + XCTAssertEqual(self.scheduledEvents.count, self.serverReceivedEvents.count); + XCTAssertEqualObjects([NSSet setWithArray:scheduledEventsByPayload.allKeys], + [NSSet setWithArray:receivedEventsByPayload.allKeys]); + + [scheduledEventsByPayload + enumerateKeysAndObjectsUsingBlock:^( + NSData *_Nonnull key, GDTCOREvent *_Nonnull scheduledEvent, BOOL *_Nonnull stop) { + GDTCOREvent *receivedEvent = receivedEventsByPayload[key]; + XCTAssertNotNil(receivedEvent); + XCTAssertEqualObjects(scheduledEvent.mappingID, receivedEvent.mappingID); + XCTAssertEqualObjects(scheduledEvent.clockSnapshot, receivedEvent.clockSnapshot); + }]; +} + +- (NSDictionary *)eventsByPayloadWithEvents: + (NSArray *)events { + NSMutableDictionary *eventsByPayload = + [NSMutableDictionary dictionaryWithCapacity:self.scheduledEvents.count]; + for (GDTCOREvent *event in events) { + eventsByPayload[event.serializedDataObjectBytes] = event; + } + + XCTAssertEqual(events.count, eventsByPayload.count, @"The event payloads must be unique"); + + return eventsByPayload; } @end diff --git a/GoogleDataTransport/GDTCCTTests/Unit/GDTCCTUploaderTest.m b/GoogleDataTransport/GDTCCTTests/Unit/GDTCCTUploaderTest.m index 2e00c6c..7e48eca 100644 --- a/GoogleDataTransport/GDTCCTTests/Unit/GDTCCTUploaderTest.m +++ b/GoogleDataTransport/GDTCCTTests/Unit/GDTCCTUploaderTest.m @@ -65,9 +65,11 @@ - (void)setUp { } - (void)tearDown { + [self.uploader waitForUploadFinishedWithTimeout:1]; self.testServer.responseCompletedBlock = nil; [self.testServer stop]; self.testStorage = nil; + self.uploader = nil; [super tearDown]; } @@ -567,14 +569,7 @@ - (void)setUpStorageExpectations { } - (void)waitForUploadOperationsToFinish:(GDTCCTUploader *)uploader { - XCTestExpectation *uploadFinishedExpectation = - [self expectationWithDescription:@"uploadFinishedExpectation"]; - dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.5 * NSEC_PER_SEC)), - uploader.uploaderQueue, ^{ - [uploadFinishedExpectation fulfill]; - XCTAssertNil(uploader.currentTask); - }); - [self waitForExpectations:@[ uploadFinishedExpectation ] timeout:1]; + XCTAssert([self.uploader waitForUploadFinishedWithTimeout:1]); } - (XCTestExpectation *)expectStorageHasEventsForTarget:(GDTCORTarget)expectedTarget diff --git a/GoogleDataTransport/GDTCCTTests/Unit/TestServer/GDTCCTTestServer.h b/GoogleDataTransport/GDTCCTTests/Unit/TestServer/GDTCCTTestServer.h index 4781e5a..15ed03d 100644 --- a/GoogleDataTransport/GDTCCTTests/Unit/TestServer/GDTCCTTestServer.h +++ b/GoogleDataTransport/GDTCCTTests/Unit/TestServer/GDTCCTTestServer.h @@ -17,6 +17,7 @@ #import #import +#import #import #import @@ -25,7 +26,7 @@ NS_ASSUME_NONNULL_BEGIN @class GCDWebServerRequest; @class GCDWebServerResponse; -typedef void (^GDTCCTTestServerRequestHandler)(GCDWebServerRequest *request, +typedef void (^GDTCCTTestServerRequestHandler)(GCDWebServerDataRequest *request, GCDWebServerResponse *_Nullable suggestedResponse, GCDWebServerCompletionBlock completionBlock); @@ -40,7 +41,7 @@ typedef void (^GDTCCTTestServerRequestHandler)(GCDWebServerRequest *request, /** Just before responding, this block will be scheduled to run on a global queue. */ @property(nonatomic, copy, nullable) void (^responseCompletedBlock) - (GCDWebServerRequest *request, GCDWebServerResponse *response); + (GCDWebServerDataRequest *request, GCDWebServerResponse *response); /** The provides an opportunity to overwrite or delay response to a request. */ @property(nonatomic, copy, nullable) GDTCCTTestServerRequestHandler requestHandler; diff --git a/GoogleDataTransport/GDTCCTTests/Unit/TestServer/GDTCCTTestServer.m b/GoogleDataTransport/GDTCCTTests/Unit/TestServer/GDTCCTTestServer.m index 8e9184d..76a3e8b 100644 --- a/GoogleDataTransport/GDTCCTTests/Unit/TestServer/GDTCCTTestServer.m +++ b/GoogleDataTransport/GDTCCTTests/Unit/TestServer/GDTCCTTestServer.m @@ -111,8 +111,8 @@ - (void)registerLogBatchPath { __auto_type __weak weakSelf = self; [self.server addHandlerForMethod:@"POST" path:@"/logBatch" - requestClass:[GCDWebServerRequest class] - asyncProcessBlock:^(__kindof GCDWebServerRequest *_Nonnull request, + requestClass:[GCDWebServerDataRequest class] + asyncProcessBlock:^(__kindof GCDWebServerDataRequest *_Nonnull request, GCDWebServerCompletionBlock _Nonnull completionBlock) { if (!weakSelf) { return; diff --git a/GoogleDataTransport/GDTCORLibrary/GDTCORFlatFileStorage+Promises.m b/GoogleDataTransport/GDTCORLibrary/GDTCORFlatFileStorage+Promises.m new file mode 100644 index 0000000..2eaafec --- /dev/null +++ b/GoogleDataTransport/GDTCORLibrary/GDTCORFlatFileStorage+Promises.m @@ -0,0 +1,100 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "GoogleDataTransport/GDTCORLibrary/Private/GDTCORFlatFileStorage+Promises.h" + +#if __has_include() +#import +#else +#import "FBLPromises.h" +#endif + +#import "GoogleDataTransport/GDTCORLibrary/Private/GDTCORUploadBatch.h" + +@implementation GDTCORFlatFileStorage (Promises) + +- (FBLPromise *> *)batchIDsForTarget:(GDTCORTarget)target { + return [FBLPromise onQueue:self.storageQueue + wrapObjectCompletion:^(FBLPromiseObjectCompletion _Nonnull handler) { + [self batchIDsForTarget:target onComplete:handler]; + }]; +} + +- (FBLPromise *)removeBatchWithID:(NSNumber *)batchID deleteEvents:(BOOL)deleteEvents { + return [FBLPromise onQueue:self.storageQueue + wrapCompletion:^(FBLPromiseCompletion _Nonnull handler) { + [self removeBatchWithID:batchID deleteEvents:deleteEvents onComplete:handler]; + }]; +} + +- (FBLPromise *)removeBatchesWithIDs:(NSSet *)batchIDs + deleteEvents:(BOOL)deleteEvents { + NSMutableArray *removeBatchPromises = + [NSMutableArray arrayWithCapacity:batchIDs.count]; + for (NSNumber *batchID in batchIDs) { + [removeBatchPromises addObject:[self removeBatchWithID:batchID deleteEvents:deleteEvents]]; + } + + return [FBLPromise onQueue:self.storageQueue all:[removeBatchPromises copy]].thenOn( + self.storageQueue, ^id(id result) { + return [FBLPromise resolvedWith:[NSNull null]]; + }); +} + +- (FBLPromise *)removeAllBatchesForTarget:(GDTCORTarget)target + deleteEvents:(BOOL)deleteEvents { + return + [self batchIDsForTarget:target].thenOn(self.storageQueue, ^id(NSSet *batchIDs) { + return [self removeBatchesWithIDs:batchIDs deleteEvents:NO]; + }); +} + +- (FBLPromise *)hasEventsForTarget:(GDTCORTarget)target { + return [FBLPromise onQueue:self.storageQueue + wrapBoolCompletion:^(FBLPromiseBoolCompletion _Nonnull handler) { + [self hasEventsForTarget:target onComplete:handler]; + }]; +} + +- (FBLPromise *)batchWithEventSelector: + (GDTCORStorageEventSelector *)eventSelector + batchExpiration:(NSDate *)expiration { + return [FBLPromise + onQueue:self.storageQueue + async:^(FBLPromiseFulfillBlock _Nonnull fulfill, FBLPromiseRejectBlock _Nonnull reject) { + [self batchWithEventSelector:eventSelector + batchExpiration:expiration + onComplete:^(NSNumber *_Nullable newBatchID, + NSSet *_Nullable batchEvents) { + if (newBatchID == nil || batchEvents == nil) { + reject([self genericRejectedPromiseErrorWithReason: + @"There are no events for the selector."]); + } else { + fulfill([[GDTCORUploadBatch alloc] initWithBatchID:newBatchID + events:batchEvents]); + } + }]; + }]; +} + +// TODO: More comprehensive error. +- (NSError *)genericRejectedPromiseErrorWithReason:(NSString *)reason { + return [NSError errorWithDomain:@"GDTCORFlatFileStorage" + code:-1 + userInfo:@{NSLocalizedFailureReasonErrorKey : reason}]; +} + +@end diff --git a/GoogleDataTransport/GDTCORLibrary/GDTCORPlatform.m b/GoogleDataTransport/GDTCORLibrary/GDTCORPlatform.m index e7bbbf6..a6207ec 100644 --- a/GoogleDataTransport/GDTCORLibrary/GDTCORPlatform.m +++ b/GoogleDataTransport/GDTCORLibrary/GDTCORPlatform.m @@ -230,8 +230,10 @@ GDTCORNetworkMobileSubtype GDTCORNetworkMobileSubTypeMessage() { code:-1 userInfo:@{NSLocalizedFailureReasonErrorKey : errorString}]; } - GDTCORLogDebug(@"Attempt to write archive. successful:%@ URL:%@ error:%@", - result ? @"YES" : @"NO", filePath, *error); + if (filePath.length > 0) { + GDTCORLogDebug(@"Attempt to write archive. successful:%@ URL:%@ error:%@", + result ? @"YES" : @"NO", filePath, *error); + } } return resultData; } diff --git a/GoogleDataTransport/GDTCORLibrary/GDTCORRegistrar.m b/GoogleDataTransport/GDTCORLibrary/GDTCORRegistrar.m index 35a5cd6..1c01c13 100644 --- a/GoogleDataTransport/GDTCORLibrary/GDTCORRegistrar.m +++ b/GoogleDataTransport/GDTCORLibrary/GDTCORRegistrar.m @@ -18,11 +18,23 @@ #import "GoogleDataTransport/GDTCORLibrary/Private/GDTCORRegistrar_Private.h" #import "GoogleDataTransport/GDTCORLibrary/Public/GoogleDataTransport/GDTCORConsoleLogger.h" +#import "GoogleDataTransport/GDTCORLibrary/Internal/GDTCORStoragePromiseAdapter.h" id _Nullable GDTCORStorageInstanceForTarget(GDTCORTarget target) { return [GDTCORRegistrar sharedInstance].targetToStorage[@(target)]; } +FOUNDATION_EXPORT +id _Nullable GDTCORStoragePromiseInstanceForTarget( + GDTCORTarget target) { + id storage = [GDTCORRegistrar sharedInstance].targetToStorage[@(target)]; + if ([storage conformsToProtocol:@protocol(GDTCORStoragePromiseProtocol)]) { + return storage; + } else { + return nil; + } +} + @implementation GDTCORRegistrar { /** Backing ivar for targetToUploader property. */ NSMutableDictionary> *_targetToUploader; @@ -67,7 +79,8 @@ - (void)registerStorage:(id)storage target:(GDTCORTarget) GDTCORRegistrar *strongSelf = weakSelf; if (strongSelf) { GDTCORLogDebug(@"Registered storage: %@ for target:%ld", storage, (long)target); - strongSelf->_targetToStorage[@(target)] = storage; + GDTCORStoragePromiseAdapter *storageAdapter = [[GDTCORStoragePromiseAdapter alloc] initWithStorage:storage]; + strongSelf->_targetToStorage[@(target)] = storageAdapter; } }); } diff --git a/GoogleDataTransport/GDTCORLibrary/GDTCORStoragePromiseAdapter.m b/GoogleDataTransport/GDTCORLibrary/GDTCORStoragePromiseAdapter.m new file mode 100644 index 0000000..ef531e4 --- /dev/null +++ b/GoogleDataTransport/GDTCORLibrary/GDTCORStoragePromiseAdapter.m @@ -0,0 +1,167 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "GoogleDataTransport/GDTCORLibrary/Internal/GDTCORStoragePromiseAdapter.h" + +#if __has_include() +#import +#else +#import "FBLPromises.h" +#endif + +#import "GoogleDataTransport/GDTCORLibrary/Private/GDTCORUploadBatch.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface GDTCORStoragePromiseAdapter () + +@property(nonatomic, readonly) dispatch_queue_t storageQueue; + +@end + +@implementation GDTCORStoragePromiseAdapter + +- (instancetype)initWithStorage:(id)storage { + self = [super init]; + if (self) { + _storage = storage; + if ([storage respondsToSelector:@selector(storageQueue)]) { + _storageQueue = [storage storageQueue]; + } else { + _storageQueue = dispatch_queue_create("GDTCORStoragePromiseAdapter", DISPATCH_QUEUE_SERIAL); + } + } + return self; +} + +- (FBLPromise *> *)batchIDsForTarget:(GDTCORTarget)target { + return [FBLPromise onQueue:self.storageQueue + wrapObjectCompletion:^(FBLPromiseObjectCompletion _Nonnull handler) { + [self.storage batchIDsForTarget:target onComplete:handler]; + }]; +} + +- (FBLPromise *)removeBatchWithID:(NSNumber *)batchID deleteEvents:(BOOL)deleteEvents { + return [FBLPromise onQueue:self.storageQueue + wrapCompletion:^(FBLPromiseCompletion _Nonnull handler) { + [self.storage removeBatchWithID:batchID deleteEvents:deleteEvents onComplete:handler]; + }]; +} + +- (FBLPromise *)removeBatchesWithIDs:(NSSet *)batchIDs + deleteEvents:(BOOL)deleteEvents { + NSMutableArray *removeBatchPromises = + [NSMutableArray arrayWithCapacity:batchIDs.count]; + for (NSNumber *batchID in batchIDs) { + [removeBatchPromises addObject:[self removeBatchWithID:batchID deleteEvents:deleteEvents]]; + } + + return [FBLPromise onQueue:self.storageQueue all:[removeBatchPromises copy]].thenOn( + self.storageQueue, ^id(id result) { + return [FBLPromise resolvedWith:[NSNull null]]; + }); +} + +- (FBLPromise *)removeAllBatchesForTarget:(GDTCORTarget)target + deleteEvents:(BOOL)deleteEvents { + return + [self batchIDsForTarget:target].thenOn(self.storageQueue, ^id(NSSet *batchIDs) { + return [self removeBatchesWithIDs:batchIDs deleteEvents:NO]; + }); +} + +- (FBLPromise *)hasEventsForTarget:(GDTCORTarget)target { + return [FBLPromise onQueue:self.storageQueue + wrapBoolCompletion:^(FBLPromiseBoolCompletion _Nonnull handler) { + [self.storage hasEventsForTarget:target onComplete:handler]; + }]; +} + +- (FBLPromise *)batchWithEventSelector: + (GDTCORStorageEventSelector *)eventSelector + batchExpiration:(NSDate *)expiration { + return [FBLPromise + onQueue:self.storageQueue + async:^(FBLPromiseFulfillBlock _Nonnull fulfill, FBLPromiseRejectBlock _Nonnull reject) { + [self.storage batchWithEventSelector:eventSelector + batchExpiration:expiration + onComplete:^(NSNumber *_Nullable newBatchID, + NSSet *_Nullable batchEvents) { + if (newBatchID == nil || batchEvents == nil) { + reject([self genericRejectedPromiseErrorWithReason: + @"There are no events for the selector."]); + } else { + fulfill([[GDTCORUploadBatch alloc] initWithBatchID:newBatchID + events:batchEvents]); + } + }]; + }]; +} + +// TODO: More comprehensive error. +- (NSError *)genericRejectedPromiseErrorWithReason:(NSString *)reason { + return [NSError errorWithDomain:@"GDTCORFlatFileStorage" + code:-1 + userInfo:@{NSLocalizedFailureReasonErrorKey : reason}]; +} + +#pragma mark - + +- (void)storeEvent:(nonnull GDTCOREvent *)event onComplete:(void (^ _Nullable)(BOOL, NSError * _Nullable))completion { + [self.storage storeEvent:event onComplete:completion]; +} + +- (void)hasEventsForTarget:(GDTCORTarget)target onComplete:(nonnull void (^)(BOOL))onComplete { + [self.storage hasEventsForTarget:target onComplete:onComplete]; +} + +- (void)batchWithEventSelector:(nonnull GDTCORStorageEventSelector *)eventSelector batchExpiration:(nonnull NSDate *)expiration onComplete:(nonnull GDTCORStorageBatchBlock)onComplete { + [self.storage batchWithEventSelector:eventSelector batchExpiration:expiration onComplete:onComplete]; +} + +- (void)removeBatchWithID:(nonnull NSNumber *)batchID deleteEvents:(BOOL)deleteEvents onComplete:(void (^ _Nullable)(void))onComplete { + [self.storage removeBatchWithID:batchID deleteEvents:deleteEvents onComplete:onComplete]; +} + +- (void)batchIDsForTarget:(GDTCORTarget)target onComplete:(nonnull void (^)(NSSet * _Nullable))onComplete { + [self.storage batchIDsForTarget:target onComplete:onComplete]; +} + +- (void)checkForExpirations { + [self.storage checkForExpirations]; +} + +- (void)storeLibraryData:(nonnull NSData *)data forKey:(nonnull NSString *)key onComplete:(nullable void (^)(NSError * _Nullable))onComplete { + [self.storage storeLibraryData:data forKey:key onComplete:onComplete]; +} + +- (void)libraryDataForKey:(nonnull NSString *)key onFetchComplete:(nonnull void (^)(NSData * _Nullable, NSError * _Nullable))onFetchComplete setNewValue:(NSData * _Nullable (^ _Nullable)(void))setValueBlock { + [self.storage libraryDataForKey:key onFetchComplete:onFetchComplete setNewValue:setValueBlock]; +} + + +- (void)removeLibraryDataForKey:(nonnull NSString *)key onComplete:(nonnull void (^)(NSError * _Nullable))onComplete { + [self.storage removeLibraryDataForKey:key onComplete:onComplete]; +} + + +- (void)storageSizeWithCallback:(nonnull void (^)(GDTCORStorageSizeBytes))onComplete { + [self.storage storageSizeWithCallback:onComplete]; +} + +@end + +NS_ASSUME_NONNULL_END diff --git a/GoogleDataTransport/GDTCORLibrary/GDTCORUploadBatch.m b/GoogleDataTransport/GDTCORLibrary/GDTCORUploadBatch.m new file mode 100644 index 0000000..7d2abe2 --- /dev/null +++ b/GoogleDataTransport/GDTCORLibrary/GDTCORUploadBatch.m @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "GoogleDataTransport/GDTCORLibrary/Private/GDTCORUploadBatch.h" + +@implementation GDTCORUploadBatch + +- (instancetype)initWithBatchID:(NSNumber *)batchID events:(NSSet *)events { + self = [super init]; + if (self) { + _batchID = batchID; + _events = events; + } + return self; +} + +@end diff --git a/GoogleDataTransport/GDTCORLibrary/Internal/GDTCORStoragePromiseAdapter.h b/GoogleDataTransport/GDTCORLibrary/Internal/GDTCORStoragePromiseAdapter.h new file mode 100644 index 0000000..0b3ee58 --- /dev/null +++ b/GoogleDataTransport/GDTCORLibrary/Internal/GDTCORStoragePromiseAdapter.h @@ -0,0 +1,31 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +#import "GoogleDataTransport/GDTCORLibrary/Internal/GDTCORStorageProtocol.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface GDTCORStoragePromiseAdapter : NSObject + +@property(nonatomic, readonly) id storage; + +- (instancetype)initWithStorage:(id)storage; + +@end + +NS_ASSUME_NONNULL_END diff --git a/GoogleDataTransport/GDTCORLibrary/Internal/GDTCORStorageProtocol.h b/GoogleDataTransport/GDTCORLibrary/Internal/GDTCORStorageProtocol.h index ade50d1..b8c5a4b 100644 --- a/GoogleDataTransport/GDTCORLibrary/Internal/GDTCORStorageProtocol.h +++ b/GoogleDataTransport/GDTCORLibrary/Internal/GDTCORStorageProtocol.h @@ -22,6 +22,9 @@ @class GDTCOREvent; @class GDTCORClock; +@class GDTCORUploadBatch; + +@class FBLPromise; NS_ASSUME_NONNULL_BEGIN @@ -118,6 +121,34 @@ typedef void (^GDTCORStorageBatchBlock)(NSNumber *_Nullable newBatchID, */ - (void)storageSizeWithCallback:(void (^)(GDTCORStorageSizeBytes storageSize))onComplete; +@optional + +/** A dispatch queue to sync storage operations. For internal use only, e.g. tests or extensions like GDTCORStoragePromiseAdapter. */ +- (dispatch_queue_t)storageQueue; + +@end + +// TODO: Consider a different place for this interface. +// TODO: Consider complete replacing block based API by promise API. +// TODO: Add API docs. +@protocol GDTCORStoragePromiseProtocol + +- (FBLPromise *> *)batchIDsForTarget:(GDTCORTarget)target; + +- (FBLPromise *)removeBatchWithID:(NSNumber *)batchID deleteEvents:(BOOL)deleteEvents; + +- (FBLPromise *)removeBatchesWithIDs:(NSSet *)batchIDs + deleteEvents:(BOOL)deleteEvents; + +- (FBLPromise *)removeAllBatchesForTarget:(GDTCORTarget)target + deleteEvents:(BOOL)deleteEvents; + +- (FBLPromise *)hasEventsForTarget:(GDTCORTarget)target; + +- (FBLPromise *)batchWithEventSelector: + (GDTCORStorageEventSelector *)eventSelector + batchExpiration:(NSDate *)expiration; + @end /** Retrieves the storage instance for the given target. @@ -128,4 +159,10 @@ typedef void (^GDTCORStorageBatchBlock)(NSNumber *_Nullable newBatchID, FOUNDATION_EXPORT id _Nullable GDTCORStorageInstanceForTarget(GDTCORTarget target); +// TODO: Ideally we should remove completion-based API and use promise-based one. Need to double +// check if it's ok. +FOUNDATION_EXPORT +id _Nullable GDTCORStoragePromiseInstanceForTarget( + GDTCORTarget target); + NS_ASSUME_NONNULL_END diff --git a/GoogleDataTransport/GDTCORLibrary/Private/GDTCORFlatFileStorage+Promises.h b/GoogleDataTransport/GDTCORLibrary/Private/GDTCORFlatFileStorage+Promises.h new file mode 100644 index 0000000..6b37679 --- /dev/null +++ b/GoogleDataTransport/GDTCORLibrary/Private/GDTCORFlatFileStorage+Promises.h @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import "GoogleDataTransport/GDTCORLibrary/Private/GDTCORFlatFileStorage.h" + +@class FBLPromise; + +NS_ASSUME_NONNULL_BEGIN + +// TODO: Docs. +// TODO: Consider a generic wrapper around an `id` object to make it more +// universal. But do we need it? +@interface GDTCORFlatFileStorage (Promises) + +- (NSError *)genericRejectedPromiseErrorWithReason:(NSString *)reason; + +@end + +NS_ASSUME_NONNULL_END diff --git a/GoogleDataTransport/GDTCORLibrary/Private/GDTCORUploadBatch.h b/GoogleDataTransport/GDTCORLibrary/Private/GDTCORUploadBatch.h new file mode 100644 index 0000000..917cd5d --- /dev/null +++ b/GoogleDataTransport/GDTCORLibrary/Private/GDTCORUploadBatch.h @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#import + +@class GDTCOREvent; + +NS_ASSUME_NONNULL_BEGIN + +@interface GDTCORUploadBatch : NSObject + +@property(nonatomic, readonly) NSNumber *batchID; +@property(nonatomic, readonly) NSSet *events; + +- (instancetype)initWithBatchID:(NSNumber *)batchID events:(NSSet *)events; + +@end + +NS_ASSUME_NONNULL_END diff --git a/GoogleDataTransport/GDTCORTests/Integration/GDTCORIntegrationTest.m b/GoogleDataTransport/GDTCORTests/Integration/GDTCORIntegrationTest.m index 8482e8d..d9beac3 100644 --- a/GoogleDataTransport/GDTCORTests/Integration/GDTCORIntegrationTest.m +++ b/GoogleDataTransport/GDTCORTests/Integration/GDTCORIntegrationTest.m @@ -169,6 +169,7 @@ - (void)testEndToEndEvent { [[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:lengthOfTestToRunInSeconds + 5]]; + XCTAssert([self.uploader waitForUploadFinishedWithTimeout:lengthOfTestToRunInSeconds]); [testServer stop]; } diff --git a/GoogleDataTransport/GDTCORTests/Integration/Helpers/GDTCORIntegrationTestUploader.h b/GoogleDataTransport/GDTCORTests/Integration/Helpers/GDTCORIntegrationTestUploader.h index 2a1253d..9960cea 100644 --- a/GoogleDataTransport/GDTCORTests/Integration/Helpers/GDTCORIntegrationTestUploader.h +++ b/GoogleDataTransport/GDTCORTests/Integration/Helpers/GDTCORIntegrationTestUploader.h @@ -30,4 +30,9 @@ */ - (instancetype)initWithServer:(GDTCORTestServer *)serverURL; +/** Spins runloop until upload finishes or timeout. + * @return YES if upload finishes, NO in the case of timeout. + */ +- (BOOL)waitForUploadFinishedWithTimeout:(NSTimeInterval)timeout; + @end diff --git a/GoogleDataTransport/GDTCORTests/Integration/Helpers/GDTCORIntegrationTestUploader.m b/GoogleDataTransport/GDTCORTests/Integration/Helpers/GDTCORIntegrationTestUploader.m index 3d658f3..1984a73 100644 --- a/GoogleDataTransport/GDTCORTests/Integration/Helpers/GDTCORIntegrationTestUploader.m +++ b/GoogleDataTransport/GDTCORTests/Integration/Helpers/GDTCORIntegrationTestUploader.m @@ -57,6 +57,11 @@ - (void)uploadTarget:(GDTCORTarget)target withConditions:(GDTCORUploadConditions arc4random_uniform(2) ? [self->_testServer.serverURL URLByAppendingPathComponent:@"log"] : [self->_testServer.serverURL URLByAppendingPathComponent:@"logBatch"]; + + // Cannot proceed if the test server is not ready or misconfigured. + if (serverURL == nil) { + return; + } NSURLSession *session = [NSURLSession sharedSession]; NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:serverURL]; request.HTTPMethod = @"POST"; @@ -79,8 +84,6 @@ - (void)uploadTarget:(GDTCORTarget)target withConditions:(GDTCORUploadConditions NSError *_Nullable error) { NSLog(@"Batch upload complete."); // Remove from the prioritizer if there were no errors. - GDTCORFatalAssert( - !error, @"There should be no errors uploading events: %@", error); if (error) { [storage removeBatchWithID:batchID deleteEvents:NO onComplete:nil]; } else { @@ -96,4 +99,17 @@ - (BOOL)readyToUploadTarget:(GDTCORTarget)target conditions:(GDTCORUploadConditi return _currentUploadTask != nil && _testServer.isRunning; } +- (BOOL)waitForUploadFinishedWithTimeout:(NSTimeInterval)timeout { + NSDate *expirationDate = [NSDate dateWithTimeIntervalSinceNow:timeout]; + while ([expirationDate compare:[NSDate date]] == NSOrderedDescending) { + if (_currentUploadTask == nil) { + return YES; + } else { + [[NSRunLoop currentRunLoop] runUntilDate:[NSDate dateWithTimeIntervalSinceNow:0.1]]; + } + } + + return NO; +} + @end diff --git a/GoogleDataTransport/generate_project.sh b/GoogleDataTransport/generate_project.sh index 59be52d..9d31e7d 100755 --- a/GoogleDataTransport/generate_project.sh +++ b/GoogleDataTransport/generate_project.sh @@ -31,4 +31,4 @@ readonly DIR="$(git rev-parse --show-toplevel)" "$DIR/GoogleDataTransport/ProtoSupport/generate_cct_protos.sh" || echo "Something went wrong generating protos."; -pod gen "$DIR/GoogleDataTransport.podspec" --auto-open --gen-directory="$DIR/gen" --platforms=${platform} --clean +pod gen "$DIR/GoogleDataTransport.podspec" --auto-open --local-sources=./ --gen-directory="$DIR/gen" --platforms=${platform} --clean diff --git a/Package.swift b/Package.swift index e968892..957c8fe 100644 --- a/Package.swift +++ b/Package.swift @@ -32,6 +32,11 @@ let package = Package( url: "https://github.com/firebase/nanopb.git", "2.30907.0" ..< "2.30908.0" ), + .package( + name: "GoogleUtilities", + url: "https://github.com/google/GoogleUtilities.git", + "7.2.1" ..< "8.0.0" + ), ], // TODO: Restructure directory structure to simplify the excludes here. targets: [ @@ -39,6 +44,7 @@ let package = Package( name: "GoogleDataTransport", dependencies: [ .product(name: "nanopb", package: "nanopb"), + .product(name: "GULEnvironment", package: "GoogleUtilities"), ], path: "GoogleDataTransport", exclude: [