2
\$\begingroup\$

I'm using the above piece of code to do some HTTP calls in parallel. This loop is running inside a NSOperation. I must throttle a bit the requests (so I don't overload the server). The code is working fine, but it doesn't seems to me the most elegant way to do it. It's OK to sleep the thread?

I'm using a two-level dispatch group, yes.

This is the main function in the NSOperation:

- (void)main
{
NSTimeInterval startOpTime = [[NSDate date] timeIntervalSince1970];    
NSFetchRequest *fetchRequest = [PartnerType fetchRequest];    
NSArray *partnerTypes = [self.parentMOC executeFetchRequest:fetchRequest error:nil];

if(partnerTypes == nil || !(partnerTypes.count > 0)) {
    XLOG_ERROR(@"%@ - Error getting PartnerTypes", kOP_DESCRIPTION);
    return;
}

dispatch_group_t groupSubOperations = dispatch_group_create();    
for(PartnerType *param in partnerTypes) {

    dispatch_group_enter(groupSubOperations);
    // Basically, in this array we store the HTTP status codes, or an error
    // (a negative integer)       
    NSArray *notifications = [self doSubOperation:param];        
    NSString *partnerTypeDesc = [param descripcion];
    // Checking if the numbers in the array are positive        
    NSInteger paramNotificationCode = [self checkNotificationCodes:notifications];
    if(paramNotificationCode >= 0) {            
        // Update last partner timestamp
        NSNumber *lastTimestamp = [self getLastTimestampForPartnerType:partnerTypeDesc];
        if(lastTimestamp) {
            NSUserDefaults *defaults = [NSUserDefaults standardUserDefaults];
            [self setUserDefaultsLastTimestamp:lastTimestamp forPartnerType:partnerType];
            [defaults synchronize];
        }
    }

    self.finalNotificationCodes = [self.finalNotificationCodes arrayByAddingObjectsFromArray:notifications];

    dispatch_group_leave(groupSubOperations);
}

dispatch_group_wait(groupSubOperations, DISPATCH_TIME_FOREVER);

// All alright?
NSInteger finalNotificationCode = [self checkNotificationCodes:self.finalNotificationCodes];
if(finalNotificationCode >= 0) {
    [self sendNotificationSuccess:finalNotificationCode timeTaken:globalDate];
} else {
    // Error getting partners
    [self sendNotificationError:finalNotificationCode];

    // Delete all BP Data
    if([@(finalNotificationCode) isEqualToNumber:ERROR_CORE_DATA]) {
        [UtilsBP deleteAllBPHeadersWithMOC:self.parentMOC];
        [UtilsBP deleteAllBPDataWithMOC:self.parentMOC];
    }

}
}

And this is the implementation of the "sub-operations":

- (NSArray *)doSubOperation:(PartnerType *)paramDefBusq
{
NSString *partnerTypeDesc = [paramDefBusq descripcion];
NSString *urlParamDetails = [paramDefBusq urlDet];

NSUserDefaults *defaults = [NSUserDefaults standardUserDefaults];

// Clear throttling cache
[defaults setObject:nil forKey:KUSERDEFAULTS_OP_GET_PARTNERS_THROTTLE];

__block NSInteger currentError = 0;

// Define which elements we need to query  
    NSNumber *syncPartnersTime = [self getUserDefaultsLastTimestampForPartnerType:partnerType];
    long syncPartnersTimestamp;
    if(syncPartnersTime != nil) {
        syncPartnersTimestamp = [syncPartnersTime longValue];
    } else {
        syncPartnersTimestamp = -1;
    }

    NSFetchRequest *codeFetchRequest = [HInterlocutor fetchRequest];
    NSEntityDescription* interlocutoresEntity = [NSEntityDescription entityForName:@"HInterlocutor" inManagedObjectContext:self.parentMOC];

    NSAttributeDescription* partnerNumAttr = (interlocutoresEntity.attributesByName)[@"code"];
    NSAttributeDescription* fechaModAttr = (interlocutoresEntity.attributesByName)[@"fechaMod"];

    [codeFetchRequest setPropertiesToFetch:@[partnerNumAttr, fechaModAttr]];
    [codeFetchRequest setPropertiesToGroupBy:@[partnerNumAttr, fechaModAttr]];

    NSPredicate *newerThanPredicate = [NSPredicate predicateWithFormat:@"fechaMod > %li", (long)syncPartnersTimestamp];
    NSPredicate *withoutBPpredicate = [NSPredicate predicateWithFormat:@"hbp = nil"];
    NSCompoundPredicate *newerThanOrWithoutBPPredicate = [NSCompoundPredicate orPredicateWithSubpredicates:@[newerThanPredicate, withoutBPpredicate]];
    NSPredicate *partnerTypePredicate = [NSPredicate predicateWithFormat:@"type = %@", partnerType];
    NSPredicate *predicate = [NSCompoundPredicate andPredicateWithSubpredicates:@[newerThanOrWithoutBPPredicate, partnerTypePredicate]];        

    [codeFetchRequest setPredicate:predicate];
    [codeFetchRequest setResultType:NSDictionaryResultType];

    NSSortDescriptor *sortDescriptor = [[NSSortDescriptor alloc] initWithKey:@"fechaMod" ascending:NO];
    [codeFetchRequest setSortDescriptors:@[sortDescriptor]];

    codesToFetch = [self.parentMOC executeFetchRequest:codeFetchRequest error:nil];

    [self setLastTimestamp:[[codesToFetch firstObject] valueForKey:@"fechaMod"] forPartnerType:partnerType];
}

unsigned long numberOfCodesToFetch = [codesToFetch count];

XLOG_INFO(@"### %@ - %@ - Elementos a pedir: %lu", kOP_DESCRIPTION, partnerType, numberOfCodesToFetch);

// Start with HTTP requests
NSInteger i = 0;
NSInteger kBatchSize = 200;
NSString *serverRequestFilter;
int numOfBatches = ((int) numberOfCodesToFetch / kBatchSize) + 1;
int currentBatch = 1;

__block NSMutableArray *notificationCodes = [NSMutableArray new];
dispatch_group_t httpDispatchGroup = dispatch_group_create();

while ((i < numberOfCodesToFetch) && currentError >= 0) {
    NSInteger batchEnd = MIN(i + kBatchSize, numberOfCodesToFetch);

    serverRequestFilter = [NSString stringWithFormat:@"BpNumber eq '%@'", codesToFetch[i][@"code"]];
    i++;
    while (i < batchEnd) {
        serverRequestFilter = [NSString stringWithFormat:@"%@ or BpNumber eq '%@'", serverRequestFilter, codesToFetch[i][@"code"]];
        i++;
    }

    NSTimeInterval startDate = [[NSDate date] timeIntervalSince1970];
    XLOG_VERBOSE(@"### %@ - %@ - OP %i/%i - HTTP START", kOP_DESCRIPTION, partnerType, currentBatch, numOfBatches);

    NSString *urlString = [NSString stringWithFormat:@"%@%@", [Utils serverName], urlParamDetails];
    urlString = [Utils addParameters:serverRequestFilter toURL:urlString];
    urlString = [urlString stringByAddingPercentEncodingWithAllowedCharacters:[NSCharacterSet URLQueryAllowedCharacterSet]];

    NSURL *url = [NSURL URLWithString:urlString];

    // Improves stability against SAP Gateway, tested and confirmed
    [self addOpToThrottleQueue:url defaults:defaults];
    [NSThread sleepForTimeInterval:[self throttleWaitInterval:defaults]];

    NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url];
    [request setHTTPMethod:@"GET"];
    [request setValue:@"application/json" forHTTPHeaderField:@"Accept"];
    [request setValue:@"gzip, deflate" forHTTPHeaderField:@"Accept-Encoding"];

    NSURLSession *session = [NSURLSession sharedSession];

    dispatch_group_enter(httpDispatchGroup);

    [[session dataTaskWithRequest:request completionHandler:^(NSData * _Nullable data, NSURLResponse * _Nullable response, NSError * _Nullable error) {

        NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *)response;

        [self removeFromThrottleQueue:httpResponse.URL defaults:defaults];

        long statusCode = (long)[httpResponse statusCode];
        NSTimeInterval endHTTPDate = [[NSDate date] timeIntervalSince1970];
        NSTimeInterval httpTime = endHTTPDate - startDate;
        XLOG_INFO(@"### %@ - %@ - OP %i/%i - HTTP END - Code: %ld, Time: %f s",kOP_DESCRIPTION, partnerType, currentBatch, numOfBatches, statusCode, httpTime);



        if(statusCode == 200) {

            if(data == nil) {
                [notificationCodes addObject:ERROR_HTTP_DATA_RECEIVED];
                currentError = [ERROR_HTTP_DATA_RECEIVED integerValue];
                dispatch_group_leave(httpDispatchGroup);
                return;
            }

            id res = [NSJSONSerialization JSONObjectWithData:data options:0 error:NULL];

            if (![res isKindOfClass:[NSDictionary class]]) {
                // Not a dictionary
                XLOG_ERROR(@"### %@ - %@ - OP %i/%i - ERROR ON DATA RECEIVED - NOT A DICTIONARY", kOP_DESCRIPTION, partnerType, currentBatch, numOfBatches);
                [notificationCodes addObject:ERROR_HTTP_DATA_RECEIVED];
                dispatch_group_leave(httpDispatchGroup);
                return;
            }               

            NSDictionary *d = ((NSDictionary *)res)[@"d"];
            NSArray *results = d[@"results"];

            @autoreleasepool {

                NSInteger partners = [results count];

                if(partners > 0) {

                    // Obtain 'Interlocutores'
                    NSArray *partnerCodes = [results valueForKey:@"BpNumber"];
                    NSFetchRequest *interlocutorRequest = [HInterlocutor fetchRequest];
                    NSPredicate *interPredicate = [NSPredicate predicateWithFormat:@"%K IN %@", @"code", partnerCodes];
                    [interlocutorRequest setPredicate:interPredicate];
                    NSError *error;
                    NSArray<HInterlocutor *> *interlocutores = [self.parentMOC executeFetchRequest:interlocutorRequest error:&error];
                    if(interlocutores.count <= 0) {
                        [notificationCodes addObject:ERROR_CORE_DATA];
                        dispatch_group_leave(httpDispatchGroup);
                        return;
                    }

                    // Save in Core Data
                    NSInteger count = partners;
                    __block NSInteger i = 0;
                    NSInteger kBatchSize = 2000;


                    [self.parentMOC performBlockAndWait:^{
                        while (i < count) {
                            NSInteger batchEnd = MIN(i + kBatchSize, count);
                            HBP *bp;
                            while (i < batchEnd) {
                                if(self.onePartner == nil) {
                                    bp = (HBP *)[NSEntityDescription insertNewObjectForEntityForName:@"HBP" inManagedObjectContext:self.parentMOC];
                                } else {
                                    NSArray *results = [DAOHBP filterByAttribute:@"partnerNum" withValue:[[self onePartner] code]];
                                    if([results count] > 0) {
                                        bp = [results firstObject];
                                    }
                                }

                                NSDictionary *rel = results[i];

                                [bp setNombreOrg:rel[@"BpName"]];
                                [bp setPartnerNum:rel[@"BpNumber"]];
                                [bp setEmail:rel[@"SmtpAddr"]];
                                [bp setTitleLetter:rel[@"TitleLetter"]];
                                [bp setReparto:rel[@"Reparto"]];
                                [bp setRazonSoc:rel[@"RazonSoc"]];
                                [bp setNif:rel[@"Nif"]];
                                [bp setPedidos:rel[@"PedidoText"]];



                        if(![self BPData_saveInCascade]) {
                            [notificationCodes addObject:ERROR_CORE_DATA];
                            if(error) {
                                XLOG_ERROR(@"%@ - %@ - Core Data Error -> %@", kOP_DESCRIPTION, partnerType, [error localizedDescription]);

                            }
                        } else {
                            [notificationCodes addObject:@(partners)];
                            [self sendNotificationSuccessPart];
                        }
                    }];

                }

                dispatch_group_leave(httpDispatchGroup);

            }

        }
    }] resume];
    currentBatch++;
}

dispatch_group_wait(httpDispatchGroup, DISPATCH_TIME_FOREVER);

// Clear throttling cache
[defaults setObject:nil forKey:KUSERDEFAULTS_OP_GET_PARTNERS_THROTTLE];

return notificationCodes;

}

For throttling, I'm keeping an NSArray of pending operations in NSUserDefaults, and depending of the count of that array at the moment of starting the request, I sleep the thread more or less, using an exponential function.

Throttling method implementation:

// Tries to balance the number of operations in paralell sent to the server
- (NSUInteger)throttleWaitInterval:(NSUserDefaults *)defaults {
    NSData *pendingOpsData = [defaults valueForKey:XXX];
    NSMutableArray *pendingOps = pendingOpsData != nil ? [NSKeyedUnarchiver unarchiveObjectWithData:pendingOpsData] : nil;

if(pendingOps == nil || ![pendingOps isKindOfClass:[NSMutableArray class]]) {
    return 1;
}

NSUInteger pendingOpsCount = [pendingOps count];
if(pendingOpsCount >= 3) {
    XLOG_VERBOSE(@"%@ - Delaying HTTP requests, %lu ops pending", kOP_DESCRIPTION, (unsigned long)pendingOpsCount);
    return (pow(pendingOpsCount,3.0)/10) + 1;
}

return 1;
}
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.