trying to speed up script that takes 14 hours

[kor]

This script loops through users and compares an AD attribute to an ini file and puts them in an AD group or takes them out of AD groups based on that attribute. It has to loop through ~68,000 users comparing against about 83 different sites. It is taking about 14 hours to complete each pass. Wondering if there is any way to speed things up or gain some sense of parallelism.

expandcollapse popup

#include <AD.au3>
#include <File.au3>
#include <Array.au3>

Global $iRunMode = 0 ;                                              ###
Global $iRunSection = 0 ;                                           ###

Global $sStudentQuery = "(&(objectCategory=person)(objectClass=user)(!userAccountControl:1.2.840.113556.1.4.803:=2)(company=stu)(!(!company=*)))", _ ; LDAP Query | person, user, not disabled, not student, company not null
$aUsers, $aDepartments, $aUserGroups, $sText, $sType, $aSchools, $aAdmins, $aStudents, $aInput, $aAllOtherAdmins, $iRunMode, $hLogFile

_AD_Open() ; open connection to Active Directory
_Students()
_AD_Close() ; close connection to Active Directory

Func _Students()
    ConsoleWrite("[INFO][BEGIN STUDENTS]")
    ConsoleWrite("[INFO][Running Student LDAP Query]")
    $aUsers = _AD_GetObjectsInOU("", $sStudentQuery, 2, "sAMAccountName,physicalDeliveryOfficeName,title", "") ; query AD (4.439s), not sorting to save 500ms
    ; $aUsers[$i][0] = sAMAccountName (username)
    ; $aUsers[$i][1] = physicalDeliveryOfficeName (site)
    ; $aUsers[$i][2] = title (grade)

    ConsoleWrite("[INFO][Reading Schools INI Section]")
    $aSchools = IniReadSection(@ScriptDir & "\GMAR-students-settings.ini", "Schools") ; read ini file for list of schools
    ; $aSchools[$i][0] = school number
    ; $aSchools[$i][1] = school group prefix

    ConsoleWrite("[INFO][Reading Student INI Section]")
    $aStudents = IniReadSection(@ScriptDir & "\GMAR-students-settings.ini", "Students") ; read ini file for student group name
    ; $aStudents[$i][0] = user type
    ; $aStudents[$i][1] = group name

    ConsoleWrite("[INFO][Running Add/Remove Checks]")
    ConsoleWrite("[VERBOSE] - Student Array contains " & UBound($aUsers) - 1 & " objects")
    ConsoleWrite("[VERBOSE] - School Array contains " & UBound($aSchools) - 1 & " objects")
    For $i = 1 To UBound($aUsers) - 1 ; loop for each user
        If Mod($i, 500) = 0 Then ConsoleWrite("Processed " & $i & " of " & UBound($aUsers) - 1 & " records") ; log entry at every 500 records
        ConsoleWrite("[VERBOSE] - User " & $i & " - " & $aUsers[$i][0])
        For $n = 1 to UBound($aSchools) - 1 ; loop for each school number
            If StringInStr($aUsers[$i][1], $aSchools[$n][0]) > 0 Then ; if users location matches school number
                If _AD_IsMemberOf($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) <> 1 Then ; if user is not already a member of the group
                    ConsoleWrite("[ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
                    If $iRunMode < 2 Then
                        _AD_AddUserToGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; add user to group
                        If @error Then ConsoleWrite("[ERROR][ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$i][1] & " - " & $aStudents[1][1])
                    EndIf
                Else
                    ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [MEMBER] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
                EndIf
            ElseIf StringInStr($aUsers[$i][1], $aSchools[$n][0]) = 0 Then ; if users location does not match school number
                If _AD_IsMemberOf($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) = 1 Then ; if user is a member of the group
                    ConsoleWrite("[REMOVE] - " & $aUsers[$i][0] & " - [FROM] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
                    If $iRunMode < 2 Then
                        _AD_RemoveUserFromGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; remove user to group
                        If @error Then ConsoleWrite("[ERROR][REMOVE] - " & $aUsers[$i][0] & " - [FROM] - " & $aSchools[$i][1] & " - " & $aStudents[1][1])
                    EndIf
                Else
                    ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [NOT MATCH] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
                EndIf
            EndIf
        Next
    Next
    ConsoleWrite("[INFO][END STUDENTS]")
EndFunc   ;==>_Students

 

example options in the INI it reads from

[schools]
101=Smith
103=Jersey
104=Washington
105=Jefferson
106=Newman

Edited September 7, 2014 by kor

[JohnOne]

Probably knock an hour off by getting rid of the console writes.

[kor]

I replaced them with consolewrite's for this example. It actually writes to a log file because I have to know what the script is doing. 

[jguinch]

For $i = 1 To UBound($aUsers) - 1 ; loop for each user
    If Mod($i, 500) = 0 Then ConsoleWrite("Processed " & $i & " of " & UBound($aUsers) - 1 & " records") ; log entry at every 500 records
    ConsoleWrite("[VERBOSE] - User " & $i & " - " & $aUsers[$i][0])
    For $n = 1 to UBound($aSchools) - 1 ; loop for each school number

        If _AD_IsMemberOf($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) <> 1 Then ; if user is already a member of the group
            If StringInStr($aUsers[$i][1], $aSchools[$n][0]) Then
                ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [MEMBER] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
            ElseIf $iRunMode < 2 Then
                _AD_RemoveUserFromGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; remove user to group
                If @error Then ConsoleWrite("[ERROR][REMOVE] - " & $aUsers[$i][0] & " - [FROM] - " & $aSchools[$i][1] & " - " & $aStudents[1][1])
            EndIf
        Else ; if user is already a not member of the group
            If NOT StringInStr($aUsers[$i][1], $aSchools[$n][0]) Then
                ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [NOT MATCH] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
            ElseIf $iRunMode < 2 Then
                ConsoleWrite("[ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
                _AD_AddUserToGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; add user to group
                If @error Then ConsoleWrite("[ERROR][ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$i][1] & " - " & $aStudents[1][1])
            EndIf
        EndIf
        
    Next
Next

With something like this, _AD_IsMemberOf will be use only one time, instead of two in your code.

Edit : I think you can skip the _AD_IsMemberOf, because _AD_AddUserToGroup and _AD_RemoveUserFromGroup already do thsi test.

Edit 2 : another way, without _AD_IsMemberOf :

If $iRunMode < 2 Then
    If StringInStr($aUsers[$i][1], $aSchools[$n][0]) Then 
        ConsoleWrite("[ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        $iRet = _AD_AddUserToGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; add user to group
        If @error Then
            If $iRet = 3
                ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [ALREADY MEMBER OF] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
            Else
                ConsoleWrite("[ERROR][ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$i][1] & " - " & $aStudents[1][1] & " (error code " & @error & ")")
            EndIf
        Else
            ConsoleWrite("[VERBOSE][OK] - " & $aUsers[$i][0] & " - [SUCCESSFULLY ADDED TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        EndIf
    Else
        ConsoleWrite("[REMOVE] - " & $aUsers[$i][0] & " - [FROM] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        $iRet = _AD_RemoveUserFromGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; remove user to group
        If @error Then
            If $iRet = 3
                ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [NOT MEMBER OF] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
            Else
                ConsoleWrite("[ERROR][REMOVE] - " & $aUsers[$i][0] & " - [FROM] - " & $aSchools[$i][1] & " - " & $aStudents[1][1] & " (error code " & @error & ")")
            EndIf
        Else
            ConsoleWrite("[VERBOSE][OK] - " & $aUsers[$i][0] & " - SUCCESSFULLY REMOVED FROM] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        EndIf
    EndIf
EndIf

Edited September 7, 2014 by jguinch

[water]

If the attributes are replicated to the Global Catalog you could drop the queries to 83 sites and simply query the GC.

[junkew]

@OP: How do you write to your logfile?

filewrite("log.txt", <yourText>)

or do you have somewhere a

$hFileLog=fileopen("log.txt",$FO_OVERWRITE)
filewrite($hFileLog, <yourText>)

The latter filewrite is much faster

First putting everything in a string with concatenate & and then writing at once to disk will speed up

[kor]

 

Edit : I think you can skip the _AD_IsMemberOf, because _AD_AddUserToGroup and _AD_RemoveUserFromGroup already do thsi test.

Edit 2 : another way, without _AD_IsMemberOf :

If $iRunMode < 2 Then
    If StringInStr($aUsers[$i][1], $aSchools[$n][0]) Then 
        ConsoleWrite("[ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        $iRet = _AD_AddUserToGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; add user to group
        If @error Then
            If $iRet = 3
                ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [ALREADY MEMBER OF] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
            Else
                ConsoleWrite("[ERROR][ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$i][1] & " - " & $aStudents[1][1] & " (error code " & @error & ")")
            EndIf
        Else
            ConsoleWrite("[VERBOSE][OK] - " & $aUsers[$i][0] & " - [SUCCESSFULLY ADDED TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        EndIf
    Else
        ConsoleWrite("[REMOVE] - " & $aUsers[$i][0] & " - [FROM] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        $iRet = _AD_RemoveUserFromGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; remove user to group
        If @error Then
            If $iRet = 3
                ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [NOT MEMBER OF] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
            Else
                ConsoleWrite("[ERROR][REMOVE] - " & $aUsers[$i][0] & " - [FROM] - " & $aSchools[$i][1] & " - " & $aStudents[1][1] & " (error code " & @error & ")")
            EndIf
        Else
            ConsoleWrite("[VERBOSE][OK] - " & $aUsers[$i][0] & " - SUCCESSFULLY REMOVED FROM] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        EndIf
    EndIf
EndIf

the only problem is, without a check first, I can't run my script it logging mode. I have a run mode that doesn't actually modify AD it just logs what it would do if it were going to modify. this way I can run it and go look at the log to see if the script is behaving as I expect

[jguinch]

OK. Maybe something like this :

expandcollapse popup

If StringInStr($aUsers[$i][1], $aSchools[$n][0]) Then 
    If $iRunMode < 2 Then
        If _AD_IsMemberOf($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) Then
            ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [ALREADY MEMBER OF] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        Else
            ConsoleWrite("[VERBOSE][ADD] - " & $aUsers[$i][0] & " - [MUST BE ADDED TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        EndIf
    Else
        ConsoleWrite("[ADD] - " & $aUsers[$i][0] & " - [TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        $iRet = _AD_AddUserToGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; add user to group
        If @error Then
            If $iRet = 3
                ConsoleWrite("[VERBOSE][SKIP] - " & $aUsers[$i][0] & " - [ALREADY MEMBER OF] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
            Else
                ConsoleWrite("[ERROR][ADD] - " & $aUsers[$i][0] & " - [ADD TO] - " & $aSchools[$i][1] & " - " & $aStudents[1][1] & " (error code " & @error & ")")
            EndIf
        Else
            ConsoleWrite("[VERBOSE][OK] - " & $aUsers[$i][0] & " - [SUCCESSFULLY ADDED TO] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        EndIf
    EndIf
Else
    If $iRunMode < 2 Then
        If _AD_IsMemberOf($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) Then
            ConsoleWrite("[VERBOSE][REMOVE] - " & $aUsers[$i][0] & " - [MUST BE REMOVED FROM] - " & $aSchools[$i][1] & " - " & $aStudents[1][1] & " (error code " & @error & ")")
        Else
            ConsoleWrite("[VERBOSE][SKIP REMOVE] - " & $aUsers[$i][0] & " - [NOT MEMBER OF] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        EndIf
    ELse
        $iRet = _AD_RemoveUserFromGroup($aSchools[$n][1] & " - " & $aStudents[1][1], $aUsers[$i][0]) ; remove user to group
        If @error Then
            If $iRet = 3
                ConsoleWrite("[VERBOSE][SKIP REMOVE] - " & $aUsers[$i][0] & " - [NOT MEMBER OF] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
            Else
                ConsoleWrite("[ERROR][REMOVE] - " & $aUsers[$i][0] & " - [REMOVE FROM] - " & $aSchools[$i][1] & " - " & $aStudents[1][1] & " (error code " & @error & ")")
            EndIf
        Else
            ConsoleWrite("[VERBOSE][OK] - " & $aUsers[$i][0] & " - SUCCESSFULLY REMOVED FROM] - " & $aSchools[$n][1] & " - " & $aStudents[1][1])
        EndIf
    EndIf
EndIf