Search the Community

After some insightful input by kylomas in >this topic, I decided to make a general resusable function using the same ideas. It handles up to 24 arrays. Zero based arrays go in, and the target array is returned ByRef. Before passing arrays to this function, you need to delete element 0 if it contains the item count; and you need to use Ubound to get the new size of the target array after using the function. If processing large arrays it is a good idea to delete the arrays you no longer need after the concatenation. The function returns the number of removed duplicates. Set case sensitivity using the second parameter 0 = case insensitive, 1 = case sensitive. ; Func _ArrayUniqueConcatenate(ByRef $aTarget, $iCasesense = 0, _ ; up to 23 more arrays can be included $a0 = 0, $a1 = 0, $a2 = 0, $a3 = 0, $a4 = 0, $a5 = 0, $a6 = 0, $a7 = 0, $a8 = 0, $a9 = 0, $a10 = 0, $a11 = 0, _ $a12 = 0, $a13 = 0, $a14 = 0, $a15 = 0, $a16 = 0, $a17 = 0, $a18 = 0, $a19 = 0, $a20 = 0, $a21 = 0, $a22 = 0) #forceref $a0, $a1, $a2, $a3, $a4, $a5, $a6, $a7, $a8, $a9, $a10, $a11, $a12, $a13, $a14, $a15, $a16, $a17, $a18, $a19, $a20, $a21, $a22 If Not IsArray($aTarget) Or UBound($aTarget, 0) <> 1 Then Return SetError(1) Local $iTotalSize = UBound($aTarget), $iItems = 0, $tVarName If $iCasesense Then For $i = 0 To $iTotalSize -1 $tVarName = "_" & StringToBinary($aTarget[$i], 2) If IsDeclared($tVarName) = -1 Then ContinueLoop Assign($tVarName, "", 1) $aTarget[$iItems] = $aTarget[$i] $iItems += 1 Next Else For $i = 0 To $iTotalSize -1 $tVarName = "_" & StringToBinary(StringLower($aTarget[$i]), 2) If IsDeclared($tVarName) = -1 Then ContinueLoop Assign($tVarName, "", 1) $aTarget[$iItems] = $aTarget[$i] $iItems += 1 Next EndIf Local $iParams = @NumParams If $iParams > 2 Then Local $aNextArray, $iBound For $i = 0 To $iParams -3 $aNextArray = Eval('a' & $i) If Not IsArray($aNextArray) Or UBound($aNextArray, 0) <> 1 Then Return SetError(2, $i +3) ; Sets @Extended to the parameter which failed $iBound = UBound($aNextArray) $iTotalSize += $iBound ReDim $aTarget[$iItems + $iBound] If $iCasesense Then For $j = 0 To $iBound -1 $tVarName = "_" & StringToBinary($aNextArray[$j], 2) If IsDeclared($tVarName) = -1 Then ContinueLoop Assign($tVarName, "", 1) $aTarget[$iItems] = $aNextArray[$j] $iItems += 1 Next Else For $j = 0 To $iBound -1 $tVarName = "_" & StringToBinary(StringLower($aNextArray[$j]), 2) If IsDeclared($tVarName) = -1 Then ContinueLoop Assign($tVarName, "", 1) $aTarget[$iItems] = $aNextArray[$j] $iItems += 1 Next EndIf Execute('_FreeMemory($a' & $i & ')') Next EndIf ReDim $aTarget[$iItems] Return $iTotalSize - $iItems ; Return the number of duplicates removed EndFunc ; _ArrayUniqueConcatenate Func _FreeMemory(ByRef $vParam) $vParam = 0 EndFunc ; In the following test, after randomly filling 24 arrays of 50000 elements (each with 2 ascii characters), the function searches (case insensitive) through 1200000 elements removing all duplicates in just a few seconds. Filling the arrays takes a few seconds to begin with (watch the SciTE console). It should hit the expected limit of 38416 possible 2 case insensitive character combinations and remove 1161584 duplicates. It takes about 13 12 seconds on my machine. Also works with unicode. ; #include <Array.au3> #include <String.au3> Global $a1[50000], $a2[50000], $a3[50000], $a4[50000], $a5[50000], $a6[50000], $a7[50000], $a8[50000], _ $a9[50000], $a10[50000], $a11[50000], $a12[50000], $a13[50000], $a14[50000], $a15[50000], $a16[50000], _ $a17[50000], $a18[50000], $a19[50000], $a20[50000], $a21[50000], $a22[50000], $a23[50000], $a24[50000] ConsoleWrite("Populating Arrays" & @LF) For $i = 1 To 24 Execute('_Fill($a' & $i & ')') Next ConsoleWrite("Starting Timer" & @LF) Local $iTimer = TimerInit() Local $ret = _ArrayUniqueConcatenate($a1, 0, $a2, $a3, $a4, $a5, $a6, $a7, $a8, $a9, $a10, $a11, $a12, $a13, $a14, $a15, $a16, $a17, $a18, $a19, $a20, $a21, $a22, $a23, $a24) ConsoleWrite("Error = " & @error & @lf & "Seconds = " & TimerDiff($iTimer)/1000 & @LF & "Unique Items = " & UBound($a1) & @LF & "Duplicates removed = " & $ret & @LF) For $i = 2 To 24 Execute('_FreeMemory($a' & $i & ')') Next _ArrayDisplay($a1) Func _FreeMemory(ByRef $vParam) $vParam = 0 EndFunc Func _Fill(ByRef $aArray) For $i = 0 To UBound($aArray) -1 $aArray[$i] = _HexToString(_RandomHexStr(4)) Next EndFunc Func _RandomHexStr($sLen) Local $sHexString = "" For $i = 1 To $sLen $sHexString &= StringRight(Hex(Random(0, 15, 1)), 1) Next Return $sHexString EndFunc ;==> _RandomHexStr Func _ArrayUniqueConcatenate(ByRef $aTarget, $iCasesense = 0, _ ; up to 23 more arrays can be included $a0 = 0, $a1 = 0, $a2 = 0, $a3 = 0, $a4 = 0, $a5 = 0, $a6 = 0, $a7 = 0, $a8 = 0, $a9 = 0, $a10 = 0, $a11 = 0, _ $a12 = 0, $a13 = 0, $a14 = 0, $a15 = 0, $a16 = 0, $a17 = 0, $a18 = 0, $a19 = 0, $a20 = 0, $a21 = 0, $a22 = 0) #forceref $a0, $a1, $a2, $a3, $a4, $a5, $a6, $a7, $a8, $a9, $a10, $a11, $a12, $a13, $a14, $a15, $a16, $a17, $a18, $a19, $a20, $a21, $a22 If Not IsArray($aTarget) Or UBound($aTarget, 0) <> 1 Then Return SetError(1) Local $iTotalSize = UBound($aTarget), $iItems = 0, $tVarName If $iCasesense Then For $i = 0 To $iTotalSize -1 $tVarName = "_" & StringToBinary($aTarget[$i], 2) If IsDeclared($tVarName) = -1 Then ContinueLoop Assign($tVarName, "", 1) $aTarget[$iItems] = $aTarget[$i] $iItems += 1 Next Else For $i = 0 To $iTotalSize -1 $tVarName = "_" & StringToBinary(StringLower($aTarget[$i]), 2) If IsDeclared($tVarName) = -1 Then ContinueLoop Assign($tVarName, "", 1) $aTarget[$iItems] = $aTarget[$i] $iItems += 1 Next EndIf Local $iParams = @NumParams If $iParams > 2 Then Local $aNextArray, $iBound For $i = 0 To $iParams -3 $aNextArray = Eval('a' & $i) If Not IsArray($aNextArray) Or UBound($aNextArray, 0) <> 1 Then Return SetError(2, $i +3) ; Sets @Extended to the parameter which failed $iBound = UBound($aNextArray) $iTotalSize += $iBound ReDim $aTarget[$iItems + $iBound] If $iCasesense Then For $j = 0 To $iBound -1 $tVarName = "_" & StringToBinary($aNextArray[$j], 2) If IsDeclared($tVarName) = -1 Then ContinueLoop Assign($tVarName, "", 1) $aTarget[$iItems] = $aNextArray[$j] $iItems += 1 Next Else For $j = 0 To $iBound -1 $tVarName = "_" & StringToBinary(StringLower($aNextArray[$j]), 2) If IsDeclared($tVarName) = -1 Then ContinueLoop Assign($tVarName, "", 1) $aTarget[$iItems] = $aNextArray[$j] $iItems += 1 Next EndIf Execute('_FreeMemory($a' & $i & ')') Next EndIf ReDim $aTarget[$iItems] Return $iTotalSize - $iItems ; Return the number of duplicates removed EndFunc ; _ArrayUniqueConcatenate