czardas Posted September 20, 2013 Author Posted September 20, 2013 (edited) I decided to try and create my own hashing function, but it's too difficult. The following code is part of that experiment and could be used as a basic checksum to test expected results after data processing of some kind. The function only hashes a sample of the data (80 bytes in fact), and the chance for collisions (although quite low) is too high for a reliable hash function. Either way it was interesting to try. this. The return value is a 40 digit hex string for data verification. There should not be any significant information leakage from knowing the checksum. The function is only suitable for use with processes where a high chance of data corruption is likely to occur somewhere in the first 1102 bytes, and it is also as yet untested. ; expandcollapse popup#include <Array.au3> Local $sString = _RandomHexStr(2*Random(1, 1200, 1)) ConsoleWrite("Binary ==> " & $sString & @LF) $sString = _GetCheckSum($sString) ConsoleWrite("Checksum ==> " & $sString & @LF) Func _GetCheckSum(ByRef $dBinary) Local $sLen = StringLen($dBinary), $dChecksum, $dNext, $iPosition, $iCount, $sString If $sLen >= 2204 Then $dChecksum = StringLeft($dBinary, 16) For $i = 1 To 9 $dChecksum &= StringMid($dBinary, $i*16 +2^$i, 16) Next ElseIf $sLen >= 160 Then $dChecksum = StringLeft($dBinary, 160) Else $dChecksum = $dBinary $iCount = 1 While 1 $dNext = StringMid($dChecksum, $iCount, 1) $iPosition = StringInStr("AC53DB42E781F960", $dNext) Switch $iPosition Case 1 To 4 $dChecksum &= _StringLoopRepeat("AC53", StringInStr("AC53", $dNext) -1, 3) Case 5 To 8 $dChecksum &= _StringLoopRepeat("DB42", StringInStr("DB42", $dNext) -1, 3) Case 9 To 12 $dChecksum &= _StringLoopRepeat("E781", StringInStr("E781", $dNext) -1, 3) Case 13 To 16 $dChecksum &= _StringLoopRepeat("F960", StringInStr("F960", $dNext) -1, 3) EndSwitch $iCount += 1 If StringLen($dChecksum) >= 160 Then ExitLoop WEnd $dChecksum = StringLeft($dChecksum, 160) EndIf $dChecksum = _Scramble($dChecksum) ; One way process $sString = "" For $j = 1 To StringLen($dChecksum) $sString &= Mod(Dec(StringMid($dChecksum, $j, 1)), 2) Next $dChecksum = "" For $i = 1 To StringLen($sString) Step 40 $dChecksum &= StringRight("0000000000" & _BaseToBase(StringMid($sString, $i, 40), 2, 16), 10) Next Return $dChecksum EndFunc ;==> _GetCheckSum Func _StringLoopRepeat($sString, $iModulate = Default, $iChars = Default) If $sString = "" Or Not IsString($sString) Then Return SetError(1, 0, "") If $iModulate = Default Then $iModulate = 0 If $iChars = Default Then $iChars = 1 If Not IsInt($iModulate) Then Return SetError(2, 0, "") If Not IsInt($iChars) Or $iChars <= 0 Then Return SetError(3, 0, "") Local $iStart, $iStringLen = StringLen($sString) $iModulate = Mod($iModulate, $iStringLen) If $iModulate >= 0 Then $iStart = $iModulate +1 Else $iStart = $iStringLen + $iModulate +1 EndIf Local $iEnd = $iStart + $iChars -1, $sNewString If $iEnd <= $iStringLen Then Return StringMid($sString, $iStart, $iChars) Else $sNewString = $sString For $i = 1 To Floor($iEnd/$iStringLen) -1 $sNewString &= $sString Next Return StringTrimLeft($sNewString, $iStart -1) & StringLeft($sString, Mod($iEnd, $iStringLen)) EndIf EndFunc ;==> _StringLoopRepeat Func _Scramble($sString) ; One way process Local $aArray = StringSplit($sString, "", 2) $sString = "" Local $iMaxIndex = UBound($aArray) Local $aTracker[$iMaxIndex][2] $iMaxIndex -= 1 For $i = 0 To $iMaxIndex $aTracker[$i][0] = $aArray[$i] & StringRight("0000" & $i, 5) $aTracker[$i][1] = $i Next _ArraySort($aTracker, 0, 0, 0, 0) ; Tracking migration in col 1 For $i = 0 To $iMaxIndex ; Replace Col 0 with original data $aTracker[$i][0] = $aArray[$i] Next _ArraySort($aTracker, 0, 0, 0, 1) ; Undo move - sort algorithm running in reverse For $i = 0 To $iMaxIndex $sString &= $aTracker[$i][0] Next Return $sString EndFunc ;==> _Scramble Func _BaseToBase($vNumber, $iBase, $iNewBase) If Not _BaseIsValid($vNumber, $iBase) Then Return SetError(1, 0, "") $vNumber = _BaseToDec($vNumber, $iBase) If @error Then Return SetError(2, 0, "") $vNumber = _DecToBase($vNumber, $iNewBase) If @error Then Return SetError(3, 0, "") Return $vNumber EndFunc ;==> _BaseToBase Func _BaseIsValid($vNumber, $iBase) If Not StringLen($vNumber) Then Return False Local $sPattern = "(?i)[^0-" Switch $iBase Case 2 To 10 $sPattern &= $iBase -1 & "]" Case 11 $sPattern &= "9A]" Case 12 To 16 $sPattern &= "9A-" & StringMid("BCDEF", $iBase -11, 1) & "]" Case Else Return False EndSwitch Return Not StringRegExp($vNumber, $sPattern) EndFunc ;==> _BaseIsValid Func _BaseToDec($vNumber, $iBase) Local $bNegative = False If StringLeft($vNumber, 1) = "-" Then $vNumber = StringTrimLeft($vNumber, 1) $bNegative = True EndIf If Not _BaseIsValid($vNumber, $iBase) Then Return SetError(1, 0, "") Local $iNewNumber = 0, $sLen = StringLen($vNumber) For $i = 1 To $sLen $iNewNumber += $iBase^($i -1)*(Dec(StringMid($vNumber, $sLen -$i +1, 1))) Next If $bNegative Then $iNewNumber = -$iNewNumber Return $iNewNumber EndFunc ;==> _BaseToDec Func _DecToBase($iDecimal, $iBase) If Not (IsInt($iDecimal) And IsInt($iBase) And $iBase >= 2 And $iBase <= 16) Then Return SetError(1, 0, "") Local $sNewNumber, $iDigit, $iPower = 1, $bNegative = False If $iDecimal < 0 Then $bNegative = True $iDecimal = Abs($iDecimal) EndIf While $iBase^$iPower <= $iDecimal $iPower += 1 WEnd For $i = $iPower -1 To 0 Step -1 $iDigit = Floor($iDecimal/($iBase^$i)) $sNewNumber &= StringRight(Hex($iDigit), 1) $iDecimal -= $iDigit*($iBase^($i)) Next If $bNegative Then $sNewNumber = "-" & $sNewNumber Return $sNewNumber EndFunc ;==> _DecToBase Func _RandomHexStr($iLen) Local $sHexString = "" For $i = 1 To Floor($iLen/7) $sHexString &= StringRight(Hex(Random(0, 0xFFFFFFF, 1)), 7) Next Return $sHexString & StringRight(Hex(Random(0, 0xFFFFFFF, 1)), Mod($iLen, 7)) EndFunc ;==> _RandomHexStr ; I just replaced the SHA-1 function (in one of my projects) with this function and it seems to work a treat. The particular processing I'm using makes it extremely unlikely to get any kind of collisions, and this quick checksum is all that is needed. I guess it can be improved though. Edited September 20, 2013 by czardas operator64  ArrayWorkshop
czardas Posted September 21, 2013 Author Posted September 21, 2013 (edited) More of the above. I think the penny dropped. No matter how clever you are, there is no way you can verify large amounts of data with 100% accuracy using a 20 byte hash string. Although I was never under such illusions, these experiments have given me a little more insight. The code in this, and the previous, post is more like a quick data scan. 'It looks alright from this end, so send the package' (or whatever). The checksum is only suitable (as I said earlier) for compairing data that stands a high chance of extensive corruption. The method can be scaled up and a larger number of bytes analysed, but the resulting hash will never be any stronger than the hash length allows. In the following version, the priciples are the same as above, but it scans smaller sections throughout the binary. With the previous example I spent most of my time trying to solve the problem of hashing less than 80 bytes (this was the most interesting part). The method for this has not changed, but the code looks very different. ; expandcollapse popup#include <Array.au3> Local $sString For $i = 1 To 64 $sString = _RandomHexStr(2 *Random(1, 512, 1)) ConsoleWrite("Binary ..... " & $sString & @LF) ConsoleWrite("Checksum ... " & _GetCheckSum($sString) & @LF & @LF) Next Func _GetCheckSum(ByRef $dBinary) Local $dChecksum, $iInt, $sTemp, $iBinaryLen = StringLen($dBinary) If $iBinaryLen >= 162 Then Local $iSkip = 1, $aFactor[12] = [3,4,6,8,9,12,16,18,24,36,48,72] ; Factors of 144 For $i = 0 To 11 If $iBinaryLen >= 144 /$aFactor[$i] *($aFactor[$i] +1) +16 Then $iInt = $i ; Data greater than 103 bytes exits the loop on the first run ExitLoop EndIf Next If $iInt = 0 Then $iSkip = Floor(($iBinaryLen -160) /48) ; Gap between scanned segments $dChecksum = StringLeft($dBinary, 16) ; Always start with the first 8 bytes For $i = 1 To 144 /$aFactor[$iInt] ; $aFactor[$iInt] is the length of each new data segment $dChecksum &= StringMid($dBinary, $i *($iSkip +$aFactor[$iInt]) +17 -$aFactor[$iInt], $aFactor[$iInt]) Next Else Local $iPosition, $dNextChar $iInt = 1 $dChecksum = $dBinary While $iBinaryLen < 160 $dNextChar = StringMid($dChecksum, $iInt, 1) ; Read from the start of the checksum $iInt += 1 $iPosition = StringInStr("CA53DB42E781F960", $dNextChar) ; Every 2 bytes has 8 bits set $sTemp = StringMid("CA53DB42E781F960", Ceiling($iPosition/4) *4 -3, 4) ; Select a 2 byte pattern $dChecksum &= _StringLoopRepeat($sTemp, StringInStr($sTemp, $dNextChar) -1, 3) $iBinaryLen += 3 ; Three hex characters were added in the previous line of code WEnd $dChecksum = StringLeft($dChecksum, 160) EndIf $sTemp = "" $dChecksum = _Scramble($dChecksum) ; one way process (mostly) For $j = 1 To 160 $sTemp &= Mod(Dec(StringMid($dChecksum, $j, 1)), 2) ; Reduction from 80 to 20 bytes Next $dChecksum = "" For $i = 1 To 160 Step 40 ; Convert binary to hexadecimal 5 bytes at a time $dChecksum &= StringRight("0000000000" & _BaseToBase(StringMid($sTemp, $i, 40), 2, 16), 10) Next Return $dChecksum EndFunc ;==> _GetCheckSum Func _StringLoopRepeat($sString, $iModulate = Default, $iChars = Default) If $sString = "" Or Not IsString($sString) Then Return SetError(1, 0, "") If $iModulate = Default Then $iModulate = 0 If $iChars = Default Then $iChars = 1 If Not IsInt($iModulate) Then Return SetError(2, 0, "") If Not IsInt($iChars) Or $iChars <= 0 Then Return SetError(3, 0, "") Local $iStart, $iStringLen = StringLen($sString) $iModulate = Mod($iModulate, $iStringLen) If $iModulate >= 0 Then $iStart = $iModulate +1 Else $iStart = $iStringLen + $iModulate +1 EndIf Local $iEnd = $iStart + $iChars -1, $sNewString If $iEnd <= $iStringLen Then Return StringMid($sString, $iStart, $iChars) Else $sNewString = $sString For $i = 1 To Floor($iEnd/$iStringLen) -1 $sNewString &= $sString Next Return StringTrimLeft($sNewString, $iStart -1) & StringLeft($sString, Mod($iEnd, $iStringLen)) EndIf EndFunc ;==> _StringLoopRepeat Func _Scramble($sString) ; Mostly a one way process Local $aArray = StringSplit($sString, "", 2) $sString = "" Local $iMaxIndex = UBound($aArray) Local $aTracker[$iMaxIndex][2] $iMaxIndex -= 1 For $i = 0 To $iMaxIndex $aTracker[$i][0] = $aArray[$i] & StringRight("0000" & $i, 5) $aTracker[$i][1] = $i Next _ArraySort($aTracker, 0, 0, 0, 0) ; Tracking migration in col 1 For $i = 0 To $iMaxIndex ; Replace Col 0 with original data $aTracker[$i][0] = $aArray[$i] Next _ArraySort($aTracker, 0, 0, 0, 1) ; Undo move - sort algorithm running in reverse For $i = 0 To $iMaxIndex $sString &= $aTracker[$i][0] Next Return $sString EndFunc ;==> _Scramble Func _BaseToBase($vNumber, $iBase, $iNewBase) If Not _BaseIsValid($vNumber, $iBase) Then Return SetError(1, 0, "") $vNumber = _BaseToDec($vNumber, $iBase) If @error Then Return SetError(2, 0, "") $vNumber = _DecToBase($vNumber, $iNewBase) If @error Then Return SetError(3, 0, "") Return $vNumber EndFunc ;==> _BaseToBase Func _BaseIsValid($vNumber, $iBase) If Not StringLen($vNumber) Then Return False Local $sPattern = "(?i)[^0-" Switch $iBase Case 2 To 10 $sPattern &= $iBase -1 & "]" Case 11 $sPattern &= "9A]" Case 12 To 16 $sPattern &= "9A-" & StringMid("BCDEF", $iBase -11, 1) & "]" Case Else Return False EndSwitch Return Not StringRegExp($vNumber, $sPattern) EndFunc ;==> _BaseIsValid Func _BaseToDec($vNumber, $iBase) Local $bNegative = False If StringLeft($vNumber, 1) = "-" Then $vNumber = StringTrimLeft($vNumber, 1) $bNegative = True EndIf If Not _BaseIsValid($vNumber, $iBase) Then Return SetError(1, 0, "") Local $iNewNumber = 0, $sLen = StringLen($vNumber) For $i = 1 To $sLen $iNewNumber += $iBase^($i -1)*(Dec(StringMid($vNumber, $sLen -$i +1, 1))) Next If $bNegative Then $iNewNumber = -$iNewNumber Return $iNewNumber EndFunc ;==> _BaseToDec Func _DecToBase($iDecimal, $iBase) If Not (IsInt($iDecimal) And IsInt($iBase) And $iBase >= 2 And $iBase <= 16) Then Return SetError(1, 0, "") Local $sNewNumber, $iDigit, $iPower = 1, $bNegative = False If $iDecimal < 0 Then $bNegative = True $iDecimal = Abs($iDecimal) EndIf While $iBase^$iPower <= $iDecimal $iPower += 1 WEnd For $i = $iPower -1 To 0 Step -1 $iDigit = Floor($iDecimal/($iBase^$i)) $sNewNumber &= StringRight(Hex($iDigit), 1) $iDecimal -= $iDigit*($iBase^($i)) Next If $bNegative Then $sNewNumber = "-" & $sNewNumber Return $sNewNumber EndFunc ;==> _DecToBase Func _RandomHexStr($iLen) Local $sHexString = "" For $i = 1 To Floor($iLen/7) $sHexString &= StringRight(Hex(Random(0, 0xFFFFFFF, 1)), 7) Next Return $sHexString & StringRight(Hex(Random(0, 0xFFFFFFF, 1)), Mod($iLen, 7)) EndFunc ;==> _RandomHexStr ; It is easy to expose the limitations of the checksum algorithm by forcing collisions to occur. Whether such collisions represent a problem depends on the intended use. At the very least, it shows the function to be working correctly for 80 bytes and above. ; Local $dCollision For $i = 1 To 20 $dCollision = "" For $j = 1 To 160 $dCollision &= Hex(2 *Random(0, 7, 1) ,1) Next ConsoleWrite("Binary ..... " & $dCollision & @LF) ConsoleWrite("Checksum ... " & _GetCheckSum($dCollision) & @LF & @LF) Next Edited October 11, 2013 by czardas operator64Â Â ArrayWorkshop
jchd Posted September 21, 2013 Posted September 21, 2013 I don't get why you don't use a proven hash. Your analogy with "100% accuracy" is hitting big numbers. It's true that cryptanalysis is making progress but the shield always comes after the bullet: "small" (e.g. doubling) the size of a key or hash causes an uncomparable increase in cryptanalysis complexity. Unless you're protecting a ketchup sauce recipe for the next million years you don't have to worry much. This wonderful site allows debugging and testing regular expressions (many flavors available). An absolute must have in your bookmarks.Another excellent RegExp tutorial. Don't forget downloading your copy of up-to-date pcretest.exe and pcregrep.exe hereRegExp tutorial: enough to get startedPCRE v8.33 regexp documentation latest available release and currently implemented in AutoIt beta. SQLitespeed is another feature-rich premier SQLite manager (includes import/export). Well worth a try.SQLite Expert (freeware Personal Edition or payware Pro version) is a very useful SQLite database manager.An excellent eBook covering almost every aspect of SQLite3: a must-read for anyone doing serious work.SQL tutorial (covers "generic" SQL, but most of it applies to SQLite as well)A work-in-progress SQLite3 tutorial. Don't miss other LxyzTHW pages!SQLite official website with full documentation (may be newer than the SQLite library that comes standard with AutoIt)
czardas Posted September 21, 2013 Author Posted September 21, 2013 (edited) Indeed. In my current implementation, the checksum may be offset by up to 512 bytes within some binary before being encrypted. I was using SHA-1 previously, but a secure hash is not really needed. Unless you already know the checksum, I don't see how you can exploit its shortcomings. 'Why else?' you ask: You learn by trying things out, and ultimately less dependencies. Using a different algorithm will not alter the chances of collisions which might occur with incorrect passwords. Odds are (16 ^ 40) to 1 against accidental collisions in both cases. Add to this the fact that the checksum has to match up with the decryption. Not much chance of that really. Edit (strike out) Further investigation shows the scramble process reduces the number of possible permutations. More about this in post 108. Edited October 11, 2013 by czardas operator64Â Â ArrayWorkshop
jchd Posted September 21, 2013 Posted September 21, 2013 I understand. The catch with any crypto-related stuff is that it only takes one very subtle mistake to ruin the most clever scheme designed by the best experts. Even barring breakthrough advances in maths theory or cryptanalysis, carefully reviewed and scrutinized designs can suddenly reveal unsuspected weaknesses. Remember the Clipper debacle, or SHA-1 we just discussed. It's true that what a crypto-expert considers broken can still be good enough for most of us mere mortals in routine operations over valueless data for limited time. But the converse is never true: what smart programmers can come up with to the best of their laymen knowledge is inevitably completely broken and can't resist serious analysis. This wonderful site allows debugging and testing regular expressions (many flavors available). An absolute must have in your bookmarks.Another excellent RegExp tutorial. Don't forget downloading your copy of up-to-date pcretest.exe and pcregrep.exe hereRegExp tutorial: enough to get startedPCRE v8.33 regexp documentation latest available release and currently implemented in AutoIt beta. SQLitespeed is another feature-rich premier SQLite manager (includes import/export). Well worth a try.SQLite Expert (freeware Personal Edition or payware Pro version) is a very useful SQLite database manager.An excellent eBook covering almost every aspect of SQLite3: a must-read for anyone doing serious work.SQL tutorial (covers "generic" SQL, but most of it applies to SQLite as well)A work-in-progress SQLite3 tutorial. Don't miss other LxyzTHW pages!SQLite official website with full documentation (may be newer than the SQLite library that comes standard with AutoIt)
czardas Posted September 21, 2013 Author Posted September 21, 2013 (edited) Wow jchd, you say some amazing things. Always interesting! If push comes to shove, I'll just remove the checksum. Edited September 22, 2013 by czardas operator64Â Â ArrayWorkshop
czardas Posted October 2, 2013 Author Posted October 2, 2013 (edited) I'm trying to figure out a way to test the strength of a password without knowing the language. Uh! I hear you say. Surely this must be impossible - but how impossible is it? Let's assume there are a minimum number of letters in an unknown alphabet. Perhaps Greek is a reasonable (lower bound) choice having just 24 letters. I'm sure there are smaller sets, but I assume the mode average is greater than 24. Eastern character sets contain thousands of characters, but most are not in general use. The question that interests me is how many characters can you fit on a keyboard? Okay you can combine keys to generate other characters too but let's think practical. Most people just type their passwords with a few keystrokes. Not knowing the language makes it impossible to give an accurate figure for password strength. Without using language detection or some complex database, the best thing I could think of was analysing the range between the highest and lowest unicode code points. The higher the range, the larger the set of characters that are likely to be included. If greater than the range for an average (Western) alphabet, the more likely both upper and lower case letters are include, or as many extra symbols as fit on a keyboard. Just some thoughts. Related Article ; Local $sTest = "abcdefghijklmNOPQRSTUVWXYZ" MsgBox(0, "Range > 26 chars", _GetCodePointRange($sTest) > 26) Func _GetCodePointRange($sString) Local $iNextChar, $iMin = -1, $iMax = -1 While StringLen($sString) $iNextChar = AscW(StringLeft($sString, 1)) If $iNextChar > $iMax Then $iMax = $iNextChar If $iMin = -1 Then $iMin = $iMax ElseIf $iNextChar < $iMin Then $iMin = $iNextChar EndIf $sString = StringRegExpReplace($sString, "\x{" & Hex($iNextChar, 4) & "}", "") WEnd Return $iMax -$iMin +1 EndFunc Edited October 2, 2013 by czardas operator64  ArrayWorkshop
czardas Posted October 11, 2013 Author Posted October 11, 2013 (edited) I set out to try and prove my claims about the function _Scramble in post 102 above and found that it may sometimes be reversible after all. Brute force tests showed that a percentage of patterns produced only had one possible input solution. These patterns are therefore reversible. Most patterns produced results that can not be reversed because different input strings shared the same output - as expected. This is the crux of the problem associated with reversing the process. Many patterns can not be produced by the function because no repeatable (two step) migration pattern exists to place the string in alphabetical order. When modulating the output on 8 unique input characters, it was found that approx 99% of the missing permutations emerged. The situation changes when the input string contains repeated characters. The entropy of the input is an influencing factor. I cannot determine whether a general solution exists to generate all the solutions of process reversal (two step migration to alphabetical order), but bear in mind I am no mathematician. I'm still bemused by the fact that I dived out of bed in the early hours of the morning several years ago to make note of this crazy scheme which appeared out of nowhere. I might be sticking my neck out once more, but it's possible that it would be easier to identify two very large prime factors than to reverse this process on a string of 100+ characters, whether the input is theoretically reversible or not. This is a problem I don't know how to solve. Edited October 11, 2013 by czardas operator64Â Â ArrayWorkshop
czardas Posted October 14, 2013 Author Posted October 14, 2013 (edited) _GetCheckSum (final version) In view of the findings above, a small change has been made. Although I doubt the code will be used by others, I believe it has some interesting features (especially the evenly spaced sampling on large amounts of data) so I'm posting the final version of the checksum. I changed the name _Scramble() to _Migration(). I never could think of an appropriate name for this function. The only real change to the code above is spinning the scrambled data sample by a number defined by the sample itself. This increases the strength of the checksum. It is still below hash standards, but less collisions will occur now than before, making it more reliable. ; expandcollapse popup#include <Array.au3> Global $sData = "ABCD" ConsoleWrite(_GetCheckSum($sData) & @CRLF) Func _GetCheckSum(ByRef $dBinary) If $dBinary = "" Or Not StringIsXDigit($dBinary) Then Return SetError(1) Local $dChecksum, $iInt, $sTemp, $iBinaryLen = StringLen($dBinary) If $iBinaryLen >= 162 Then Local $iSkip = 1, $aFactor[12] = [3,4,6,8,9,12,16,18,24,36,48,72] ; Factors of 144 ; Determine optimal scan segment size - no less than 3 hexadecimal characters For $i = 0 To 11 If $iBinaryLen >= 144 /$aFactor[$i] *($aFactor[$i] +1) +16 Then $iInt = $i ; Data greater than 103 bytes exits the loop on the first run ExitLoop EndIf Next If $iInt = 0 Then $iSkip = Floor(($iBinaryLen -160) /48) ; Gap between scanned segments $dChecksum = StringLeft($dBinary, 16) ; Always start with the first 8 bytes For $i = 1 To 144 /$aFactor[$iInt] ; $aFactor[$iInt] is the length of each new data segment $dChecksum &= StringMid($dBinary, $i *($iSkip +$aFactor[$iInt]) +17 -$aFactor[$iInt], $aFactor[$iInt]) Next Else Local $iPosition, $dNextChar $iInt = 1 $dChecksum = $dBinary While $iBinaryLen < 160 $dNextChar = StringMid($dChecksum, $iInt, 1) ; Read from the start of the checksum $iInt += 1 $iPosition = StringInStr("CA53DB42E781F960", $dNextChar) ; Every 2 bytes has 8 bits set $sTemp = StringMid("CA53DB42E781F960", Ceiling($iPosition/4) *4 -3, 4) ; Select a 2 byte pattern $dChecksum &= _StringLoopRepeat($sTemp, StringInStr($sTemp, $dNextChar) -1, 3) $iBinaryLen += 3 ; Three hex characters were added in the previous line of code WEnd $dChecksum = StringLeft($dChecksum, 160) EndIf ; Major improvement $dChecksum = _StringModulate(_Migration($dChecksum), _NibbleSum($dChecksum)) ; Mostly a one way process $sTemp = "" For $j = 1 To 160 $sTemp &= Mod(Dec(StringMid($dChecksum, $j, 1)), 2) ; Reduction from 80 to 20 bytes Next $dChecksum = "" For $i = 1 To 160 Step 40 ; Convert binary to hexadecimal 40 bits at a time $dChecksum &= StringRight("0000000000" & _BaseToBase(StringMid($sTemp, $i, 40), 2, 16), 10) Next Return $dChecksum EndFunc ;==> _GetCheckSum Func _NibbleSum($dHex) Local $iSum = 0 For $j = 1 To StringLen($dHex) $iSum += Dec(StringMid($dHex, $j, 1)) Next Return $iSum EndFunc ; _NibbleSum Func _StringLoopRepeat($sString, $iModulate = Default, $iChars = Default) If $sString = "" Or Not IsString($sString) Then Return SetError(1, 0, "") If $iModulate = Default Then $iModulate = 0 If $iChars = Default Then $iChars = 1 If Not IsInt($iModulate) Then Return SetError(2, 0, "") If Not IsInt($iChars) Or $iChars <= 0 Then Return SetError(3, 0, "") Local $iStart, $iStringLen = StringLen($sString) $iModulate = Mod($iModulate, $iStringLen) If $iModulate >= 0 Then $iStart = $iModulate +1 Else $iStart = $iStringLen + $iModulate +1 EndIf Local $iEnd = $iStart + $iChars -1, $sNewString If $iEnd <= $iStringLen Then Return StringMid($sString, $iStart, $iChars) Else $sNewString = $sString For $i = 1 To Floor($iEnd/$iStringLen) -1 $sNewString &= $sString Next Return StringTrimLeft($sNewString, $iStart -1) & StringLeft($sString, Mod($iEnd, $iStringLen)) EndIf EndFunc ;==> _StringLoopRepeat Func _StringModulate($sString, $iModulate) If $sString = "" Or Not IsString($sString) Then Return SetError(1, 0, "") If Not IsInt($iModulate) Then Return SetError(2, 0, "") Local $iStringLen = StringLen($sString) $iModulate = Mod($iModulate, $iStringLen) If $iModulate = 0 Then Return $sString If $iModulate > 0 Then Return StringRight($sString, $iStringLen -$iModulate) & StringLeft($sString, $iModulate) Else Return StringRight($sString, -$iModulate) & StringLeft($sString, $iStringLen +$iModulate) EndIf EndFunc ;==> _StringModulate Func _Migration($sString) ; Mostly a one way process Local $aArray = StringSplit($sString, "", 2) $sString = "" Local $iMaxIndex = UBound($aArray) Local $aTracker[$iMaxIndex][2] $iMaxIndex -= 1 For $i = 0 To $iMaxIndex $aTracker[$i][0] = $aArray[$i] & StringRight("00000" & $i, 6) $aTracker[$i][1] = $i Next _ArraySort($aTracker, 0, 0, 0, 0) ; Tracking migration in col 1 (UDF ==> _ArraySort) For $i = 0 To $iMaxIndex ; Replace Col 0 with original data $aTracker[$i][0] = $aArray[$i] Next _ArraySort($aTracker, 0, 0, 0, 1) ; Undo move - sort algorithm running in reverse (UDF ==> _ArraySort) For $i = 0 To $iMaxIndex $sString &= $aTracker[$i][0] Next Return $sString EndFunc ;==> _Migration ; Functions from 'UDF Suite' ==> Func _BaseToBase($vNumber, $iBase, $iNewBase) If Not _BaseIsValid($vNumber, $iBase) Then Return SetError(1, 0, "") $vNumber = _BaseToDec($vNumber, $iBase) If @error Then Return SetError(2, 0, "") $vNumber = _DecToBase($vNumber, $iNewBase) If @error Then Return SetError(3, 0, "") Return $vNumber EndFunc ;==> _BaseToBase Func _BaseToDec($vNumber, $iBase) Local $bNegative = False If StringLeft($vNumber, 1) = "-" Then $vNumber = StringTrimLeft($vNumber, 1) $bNegative = True EndIf If Not _BaseIsValid($vNumber, $iBase) Then Return SetError(1, 0, "") Local $iNewNumber = 0, $sLen = StringLen($vNumber) For $i = 1 To $sLen $iNewNumber += $iBase^($i -1)*(Dec(StringMid($vNumber, $sLen -$i +1, 1))) Next If $bNegative Then $iNewNumber = -$iNewNumber Return $iNewNumber EndFunc ;==> _BaseToDec Func _DecToBase($iDecimal, $iBase) If Not (IsInt($iDecimal) And IsInt($iBase) And $iBase >= 2 And $iBase <= 16) Then Return SetError(1, 0, "") Local $sNewNumber, $iDigit, $iPower = 1, $bNegative = False If $iDecimal < 0 Then $bNegative = True $iDecimal = Abs($iDecimal) EndIf While $iBase^$iPower <= $iDecimal $iPower += 1 WEnd For $i = $iPower -1 To 0 Step -1 $iDigit = Floor($iDecimal/($iBase^$i)) $sNewNumber &= StringRight(Hex($iDigit), 1) $iDecimal -= $iDigit*($iBase^($i)) Next If $bNegative Then $sNewNumber = "-" & $sNewNumber Return $sNewNumber EndFunc ;==> _DecToBase Func _BaseIsValid($vNumber, $iBase) If Not StringLen($vNumber) Then Return False Local $sPattern = "(?i)[^0-" Switch $iBase Case 2 To 10 $sPattern &= $iBase -1 & "]" Case 11 $sPattern &= "9A]" Case 12 To 16 $sPattern &= "9A-" & StringMid("BCDEF", $iBase -11, 1) & "]" Case Else Return False EndSwitch Return Not StringRegExp($vNumber, $sPattern) EndFunc ;==> _BaseIsValid ; The catch with any crypto-related stuff is that it only takes one very subtle mistake to ruin the most clever scheme designed by the best experts. Even barring breakthrough advances in maths theory or cryptanalysis, carefully reviewed and scrutinized designs can suddenly reveal unsuspected weaknesses. Â And I thought adding random junk was a good idea. It's totally stupid in fact! Edited October 14, 2013 by czardas operator64Â Â ArrayWorkshop
czardas Posted October 25, 2013 Author Posted October 25, 2013 (edited) Using 'Private Use Zone' Unicode for delimiters is an idea. It works perfectly in my example but I haven't run many tests yet. I believe this to be in accordance with intended use. ; #include <Array.au3> Local $sStr = "346080430654032102467430124520330370676041914036542087850643" Local $sDelim = _GetDelim($sStr) $sStr = StringReplace($sStr, "0", $sDelim & "0") ; Just an example Local $aSplit = StringSplit($sStr, $sDelim, 2) ; Split before zero _ArrayDisplay($aSplit, "Delim = Chr(" & AscW($sDelim) & ")") Func _GetDelim($sString) Local $sDelim For $i = 57344 To 63743 ; Private Use Range $sDelim = ChrW($i) If Not StringInStr($sString, $sDelim) Then Return $sDelim Next Return SetError(1, 0, "") ; String is unsuitable and obscure EndFunc Edited October 25, 2013 by czardas operator64Â Â ArrayWorkshop
czardas Posted March 13, 2014 Author Posted March 13, 2014 (edited) Work in progress - Store guitar fingerings (within a 5 fret stretch) using base 36. Open strings may, or may not, be muted. Example: JLZ 313355 |•|||| |||||| •|••|| |||||| ||||•• ; expandcollapse popupMsgBox(0, "", _Base36ToTAB("JLZ")) ; 313355 ; Convert 6 digit Guitar TAB to 3 base 36 characters [0-9A-Z] Func _TABToBase36($sChordTAB) Static $a36 = _Base36() Local $aFingers = _StringEqualSplit($sChordTAB, 2) ; Split TAB to 3 equal length segments Local $sBase36 = "" For $i = 0 To 2 $sBase36 &= $a36[_BaseToDec($aFingers[$i], 6)] Next Return $sBase36 EndFunc ; Convert 3 base 36 characters [0-9A-Z] to 6 digit Guitar TAB Func _Base36ToTAB($sBase36) Local $sNextChar, $sChordTAB For $i = 1 To 3 $sNextChar = StringMid($sBase36, $i, 1) If StringIsAlpha($sNextChar) Then $sNextChar = Asc(StringUpper($sNextChar)) -55 $sNextChar = _DecToBase($sNextChar, 6) If $sNextChar < 6 Then $sNextChar = "0" & $sNextChar $sChordTAB &= $sNextChar Next Return $sChordTAB EndFunc Func _BaseToDec($vNumber, $iBase) Local $bNegative = False If StringLeft($vNumber, 1) = "-" Then $vNumber = StringTrimLeft($vNumber, 1) $bNegative = True EndIf Local $iNewNumber = 0, $sLen = StringLen($vNumber) For $i = 1 To $sLen $iNewNumber += $iBase^($i -1)*(Dec(StringMid($vNumber, $sLen -$i +1, 1))) Next If $bNegative Then $iNewNumber = -$iNewNumber Return $iNewNumber EndFunc ;==> _BaseToDec Func _DecToBase($iDecimal, $iBase) Local $sNewNumber, $iDigit, $iPower = 1, $bNegative = False If $iDecimal < 0 Then $bNegative = True $iDecimal = Abs($iDecimal) EndIf While $iBase^$iPower <= $iDecimal $iPower += 1 WEnd For $i = $iPower -1 To 0 Step -1 $iDigit = Floor($iDecimal/($iBase^$i)) $sNewNumber &= StringRight(Hex($iDigit), 1) $iDecimal -= $iDigit*($iBase^($i)) Next If $bNegative Then $sNewNumber = "-" & $sNewNumber Return $sNewNumber EndFunc ;==> _DecToBase ; Splits a string into equal length segments ==> czardas Func _StringEqualSplit($sString, $iNumChars) Return StringRegExp($sString, "(?s).{1," & $iNumChars & "}", 3) EndFunc ; _StringEqualSplit ; Create a 36 element array of values 0-Z Func _Base36() Local $a36[36] For $i = 0 To 9 $a36[$i] = $i Next For $i = 10 To 35 $a36[$i] = Chr($i +55) Next Return $a36 EndFunc ; Some error checks will be removed for optimization. The system only stores guitar fingerings and is intended purely as a form of compression. Simple yet effective! Edited March 13, 2014 by czardas Danyfirex 1 operator64  ArrayWorkshop
JohnOne Posted March 13, 2015 Posted March 13, 2015 (edited) Shuffle the elements within a Multidimensional ArrayI was trying to be clever using a recursive function for this, but it got too confusing. So I have created a small monster instead. This function will shuffle a multidimensional array of up to 11 dimensions. When I say shuffle - I mean any element can end up anywhere in any dimension. expandcollapse popup#include <Array.au3> ; For _ArrayDisplay #region ; Example Global $aGrid[4][4] = [[0,1,2,3],[4,"",5,6],[7,8,0,9],["a","b","c",""]] _ArrayDisplay($aGrid) For $i = 0 To 9 _ArrayShuffleMultiDim($aGrid) _ArrayDisplay($aGrid) Next #endregion Func _ArrayShuffleMultiDim(ByRef $aArray) If IsArray($aArray) = 0 Then Return SetError (1) Local $iDim = UBound($aArray, 0) ; Get No dimensions If $iDim > 11 Then Return SetError (2) Local $a[$iDim +1], $aiBound[12] = [0,0,0,0,0,0,0,0,0,0,0,0] For $i = 1 To $iDim ; For each dimensionn $aiBound[$i] = UBound($aArray, $i) -1 ; Get dimension size Next For $i = 0 To $aiBound[1] For $j = 0 To $aiBound[2] For $k = 0 To $aiBound[3] For $l = 0 To $aiBound[4] For $m = 0 To $aiBound[5] For $n = 0 To $aiBound[6] For $o = 0 To $aiBound[7] For $p = 0 To $aiBound[8] For $q = 0 To $aiBound[9] For $r = 0 To $aiBound[10] For $s = 0 To $aiBound[11] For $z = 1 To $iDim ; Get random values for each dimension. $a[$z] = Random(0, $aiBound[$z], 1) Next __Swap($aArray, $iDim, $a, $i, $j, $k, $l, $m, $n, $o, $p, $q, $r, $s) Next Next Next Next Next Next Next Next Next Next Next EndFunc Func __Swap(ByRef $aArray, $iDim, $a, $i, $j, $k, $l, $m, $n, $o, $p, $q, $r, $s) Local $vTemp Switch $iDim Case 1 $vTemp = $aArray[$i] $aArray[$i] = $aArray[$a[1]] $aArray[$a[1]] = $vTemp Case 2 $vTemp = $aArray[$i][$j] $aArray[$i][$j] = $aArray[$a[1]][$a[2]] $aArray[$a[1]][$a[2]] = $vTemp Case 3 $vTemp = $aArray[$i][$j][$k] $aArray[$i][$j][$k] = $aArray[$a[1]][$a[2]][$a[3]] $aArray[$a[1]][$a[2]][$a[3]] = $vTemp Case 4 $vTemp = $aArray[$i][$j][$k][$l] $aArray[$i][$j][$k][$l] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]] = $vTemp Case 5 $vTemp = $aArray[$i][$j][$k][$l][$m] $aArray[$i][$j][$k][$l][$m] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]] = $vTemp Case 6 $vTemp = $aArray[$i][$j][$k][$l][$m][$n] $aArray[$i][$j][$k][$l][$m][$n] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]] = $vTemp Case 7 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o] $aArray[$i][$j][$k][$l][$m][$n][$o] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]] = $vTemp Case 8 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o][$p] $aArray[$i][$j][$k][$l][$m][$n][$o][$p] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]] = $vTemp Case 9 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q] $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]] = $vTemp Case 10 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r] $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]][$a[10]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]][$a[10]] = $vTemp Case 11 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r][$s] $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r][$s] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]][$a[10]][$a[11]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]][$a[10]][$a[11]] = $vTemp EndSwitch EndFuncOther than setting up a game of battleships, I can't think of many uses for shuffling a multidimensional array. If anyone can suggest any potential uses for this I would love to hear your ideas.  RE: expandcollapse popup#include <array.au3> Local $a2D[3][3] = [["a", "b", "c"],["d", "e", "f"],["g", "h", "i"]] _ArrayDisplay($a2D) _MDA_Shuffle($a2D) _ArrayDisplay($a2D) Func _MDA_Shuffle(ByRef $array) Local $iRows = UBound($array) Local $iCols = UBound($array, 2) If $iRows * $iCols >= 16777216 Then Return SetError(1) EndIf Local $a1D[$iRows * $iCols] ;Local $aTmp[$iRows][$iCols] Local $iIndex1D = 0 For $iR = 0 To $iRows - 1 For $iC = 0 To $iCols - 1 $a1D[$iIndex1D] = $array[$iR][$iC] $iIndex1D += 1 Next Next _ArrayShuffle($a1D) $iIndex1D = 0 For $iR = 0 To $iRows - 1 For $iC = 0 To $iCols - 1 $array[$iR][$iC] = $a1D[$iIndex1D] $iIndex1D += 1 Next Next EndFunc ;==>_MDA_Shuffle EDIT: forgot error checking.EDIT2: * for ^ Edited March 13, 2015 by JohnOne AutoIt Absolute Beginners  Require a serial  Pause Script  Video Tutorials by Morthawt  ipify Monkey's are, like, natures humans.
czardas Posted March 13, 2015 Author Posted March 13, 2015 (edited) It looks simple enough in theory, but once you start adding more dimensions the code will end up looking just as ridiculous as the code I wrote using that method. Thanks for posting it because it's certainly easier to get your head around. I'm skeptical it would be as fast though: conversion to a 1D array is an extra step. Are you sure about your added checks? $iRows ^ $iCols Edited March 13, 2015 by czardas operator64Â Â ArrayWorkshop
JohnOne Posted March 13, 2015 Posted March 13, 2015 VAR_SUBSCRIPT_ELEMENTS 16,777,216 Maximum number of elements for an array.Not sure about speed.Can't thing of a quick way try create a massive 2d array AutoIt Absolute Beginners  Require a serial  Pause Script  Video Tutorials by Morthawt  ipify Monkey's are, like, natures humans.
czardas Posted March 13, 2015 Author Posted March 13, 2015 (edited) You are using the power operator in the wrong context. It should be the multiplication operator (*). $iRows * $iCols Most of the loops in the code I created will only run once. That's the trick of looping through extra dimensions just in case there are more than two. It's rare to see more than 3 dimensions used so only the final loops are likely to repeat. The rest are simply looping For zero To zero times, which is the same as once. Edited March 13, 2015 by czardas operator64Â Â ArrayWorkshop
JohnOne Posted March 13, 2015 Posted March 13, 2015 Oops, just a typo on forum. The way I've done it is no slouch, like. expandcollapse popup#include <array.au3> Global $aGrid[20][10] = [[0,1,2,3,3,4,5,6,7,8],[4,"",5,6,2,6,5,7,8,9],[7,8,0,9,6,6,6,6,6,6],["a","b","c","",3,2,4,0,7,5] _ ,["a","b","c","",3,2,4,0,7,5],["a","b","c","",3,2,4,0,7,5],["a","b","c","",3,2,4,0,7,5],["a","b","c","",3,2,4,0,7,5] _ ,["a","b","c","",3,2,4,0,7,5],["a","b","c","",3,2,4,0,7,5],["a","b","c","",3,2,4,0,7,5],["a","b","c","",3,2,4,0,7,5] _ ,["a","b","c","",3,2,4,0,7,5],["a","b","c","",3,2,4,0,7,5],["a","b","c","",3,2,4,0,7,5],["a","b","c","",3,2,4,0,7,5] _ ,["a","b","c","",3,2,4,0,7,5],["a","b","c","",3,2,4,0,7,5],["a","b","c","",3,2,4,0,7,5],["a","b","c","",3,2,4,0,7,5]] _ArrayDisplay($aGrid) $loops = 1000 $Timer = TimerInit() For $i = 0 To $loops _MDA_Shuffle($aGrid) ;_ArrayDisplay($aGrid) Next ConsoleWrite("_MDA_Shuffle " & TimerDiff($Timer) & @LF) $Timer = TimerInit() For $i = 0 To $loops _ArrayShuffleMultiDim($aGrid) ;_ArrayDisplay($aGrid) Next ConsoleWrite("_ArrayShuffleMultiDim " & TimerDiff($Timer) & @LF) Func _ArrayShuffleMultiDim(ByRef $aArray) If IsArray($aArray) = 0 Then Return SetError (1) Local $iDim = UBound($aArray, 0) ; Get No dimensions If $iDim > 11 Then Return SetError (2) Local $a[$iDim +1], $aiBound[12] = [0,0,0,0,0,0,0,0,0,0,0,0] For $i = 1 To $iDim ; For each dimensionn $aiBound[$i] = UBound($aArray, $i) -1 ; Get dimension size Next For $i = 0 To $aiBound[1] For $j = 0 To $aiBound[2] For $k = 0 To $aiBound[3] For $l = 0 To $aiBound[4] For $m = 0 To $aiBound[5] For $n = 0 To $aiBound[6] For $o = 0 To $aiBound[7] For $p = 0 To $aiBound[8] For $q = 0 To $aiBound[9] For $r = 0 To $aiBound[10] For $s = 0 To $aiBound[11] For $z = 1 To $iDim ; Get random values for each dimension. $a[$z] = Random(0, $aiBound[$z], 1) Next __Swap($aArray, $iDim, $a, $i, $j, $k, $l, $m, $n, $o, $p, $q, $r, $s) Next Next Next Next Next Next Next Next Next Next Next EndFunc Func __Swap(ByRef $aArray, $iDim, $a, $i, $j, $k, $l, $m, $n, $o, $p, $q, $r, $s) Local $vTemp Switch $iDim Case 1 $vTemp = $aArray[$i] $aArray[$i] = $aArray[$a[1]] $aArray[$a[1]] = $vTemp Case 2 $vTemp = $aArray[$i][$j] $aArray[$i][$j] = $aArray[$a[1]][$a[2]] $aArray[$a[1]][$a[2]] = $vTemp Case 3 $vTemp = $aArray[$i][$j][$k] $aArray[$i][$j][$k] = $aArray[$a[1]][$a[2]][$a[3]] $aArray[$a[1]][$a[2]][$a[3]] = $vTemp Case 4 $vTemp = $aArray[$i][$j][$k][$l] $aArray[$i][$j][$k][$l] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]] = $vTemp Case 5 $vTemp = $aArray[$i][$j][$k][$l][$m] $aArray[$i][$j][$k][$l][$m] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]] = $vTemp Case 6 $vTemp = $aArray[$i][$j][$k][$l][$m][$n] $aArray[$i][$j][$k][$l][$m][$n] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]] = $vTemp Case 7 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o] $aArray[$i][$j][$k][$l][$m][$n][$o] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]] = $vTemp Case 8 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o][$p] $aArray[$i][$j][$k][$l][$m][$n][$o][$p] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]] = $vTemp Case 9 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q] $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]] = $vTemp Case 10 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r] $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]][$a[10]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]][$a[10]] = $vTemp Case 11 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r][$s] $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r][$s] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]][$a[10]][$a[11]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]][$a[10]][$a[11]] = $vTemp EndSwitch EndFunc Func _MDA_Shuffle(ByRef $array) Local $iRows = UBound($array) Local $iCols = UBound($array, 2) If $iRows * $iCols >= 16777216 Then Return SetError(1) EndIf Local $a1D[$iRows * $iCols] Local $iIndex1D = 0 For $iR = 0 To $iRows - 1 For $iC = 0 To $iCols - 1 $a1D[$iIndex1D] = $array[$iR][$iC] $iIndex1D += 1 Next Next _ArrayShuffle($a1D) $iIndex1D = 0 For $iR = 0 To $iRows - 1 For $iC = 0 To $iCols - 1 $array[$iR][$iC] = $a1D[$iIndex1D] $iIndex1D += 1 Next Next EndFunc ;==>_MDA_Shuffle czardas 1 AutoIt Absolute Beginners  Require a serial  Pause Script  Video Tutorials by Morthawt  ipify Monkey's are, like, natures humans.
czardas Posted March 13, 2015 Author Posted March 13, 2015 (edited) Indeed it is faster. It's a while ago since I wrote this, and _ArrayShuffle() hadn't been included in the language at the time. That has nothing to do with where the speed is being lost though because it's exactly the same method. I was a bit confused about this. Maybe it's the Switch statements that are slowing it down. I think it should still probably be done without creating a new array though, but this might only be correct in theory. The problem is that I'm not quite sure how to implement your method with a varied number of dimensions. The test below is specidic for 11 dimensions only. Creating a second array is still faster than my original code. I'll have to think about this. ; expandcollapse popup#include <array.au3> Global $aDim11[2][2][2][2][2][2][2][2][2][2][2] ; 11 dimension array ; Populate the array For $i = 0 To 1 For $j = 0 To 1 For $k = 0 To 1 For $l = 0 To 1 For $m = 0 To 1 For $n = 0 To 1 For $o = 0 To 1 For $p = 0 To 1 For $q = 0 To 1 For $r = 0 To 1 For $s = 0 To 1 $aDim11[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r][$s] = $i &$j &$k &$l &$m &$n &$o &$p &$q &$r &$s Next Next Next Next Next Next Next Next Next Next Next $loops = 50 $Timer = TimerInit() For $i = 0 To $loops _Shuffle_11Dim($aDim11) Next ConsoleWrite("_Shuffle_11Dim " & TimerDiff($Timer) & @LF) $Timer = TimerInit() For $i = 0 To $loops _ArrayShuffleMultiDim($aDim11) Next ConsoleWrite("_ArrayShuffleMultiDim " & TimerDiff($Timer) & @LF) Func _ArrayShuffleMultiDim(ByRef $aArray) If IsArray($aArray) = 0 Then Return SetError (1) Local $iDim = UBound($aArray, 0) ; Get No dimensions If $iDim > 11 Then Return SetError (2) Local $a[$iDim +1], $aiBound[12] = [0,0,0,0,0,0,0,0,0,0,0,0] For $i = 1 To $iDim ; For each dimensionn $aiBound[$i] = UBound($aArray, $i) -1 ; Get dimension size Next For $i = 0 To $aiBound[1] For $j = 0 To $aiBound[2] For $k = 0 To $aiBound[3] For $l = 0 To $aiBound[4] For $m = 0 To $aiBound[5] For $n = 0 To $aiBound[6] For $o = 0 To $aiBound[7] For $p = 0 To $aiBound[8] For $q = 0 To $aiBound[9] For $r = 0 To $aiBound[10] For $s = 0 To $aiBound[11] For $z = 1 To $iDim ; Get random values for each dimension. $a[$z] = Random(0, $aiBound[$z], 1) Next __Swap($aArray, $iDim, $a, $i, $j, $k, $l, $m, $n, $o, $p, $q, $r, $s) Next Next Next Next Next Next Next Next Next Next Next EndFunc Func __Swap(ByRef $aArray, $iDim, $a, $i, $j, $k, $l, $m, $n, $o, $p, $q, $r, $s) Local $vTemp Switch $iDim Case 1 $vTemp = $aArray[$i] $aArray[$i] = $aArray[$a[1]] $aArray[$a[1]] = $vTemp Case 2 $vTemp = $aArray[$i][$j] $aArray[$i][$j] = $aArray[$a[1]][$a[2]] $aArray[$a[1]][$a[2]] = $vTemp Case 3 $vTemp = $aArray[$i][$j][$k] $aArray[$i][$j][$k] = $aArray[$a[1]][$a[2]][$a[3]] $aArray[$a[1]][$a[2]][$a[3]] = $vTemp Case 4 $vTemp = $aArray[$i][$j][$k][$l] $aArray[$i][$j][$k][$l] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]] = $vTemp Case 5 $vTemp = $aArray[$i][$j][$k][$l][$m] $aArray[$i][$j][$k][$l][$m] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]] = $vTemp Case 6 $vTemp = $aArray[$i][$j][$k][$l][$m][$n] $aArray[$i][$j][$k][$l][$m][$n] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]] = $vTemp Case 7 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o] $aArray[$i][$j][$k][$l][$m][$n][$o] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]] = $vTemp Case 8 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o][$p] $aArray[$i][$j][$k][$l][$m][$n][$o][$p] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]] = $vTemp Case 9 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q] $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]] = $vTemp Case 10 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r] $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]][$a[10]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]][$a[10]] = $vTemp Case 11 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r][$s] $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r][$s] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]][$a[10]][$a[11]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]][$a[10]][$a[11]] = $vTemp EndSwitch EndFunc ; Specific 11 dimension shuffle based on JohnOne's code Func _Shuffle_11Dim(ByRef $aArray) Local $iDim = 11 ; Get No dimensions - Let's cheat here by giving the answer = 11. Local $a[$iDim +1], $aiBound[12] = [0,0,0,0,0,0,0,0,0,0,0,0] Local $iTemp = 1, $iBound = 0 For $i = 1 To $iDim ; For each dimension $iBound = UBound($aArray, $i) ; Get dimension size $aiBound[$i] = $iBound -1 $iTemp *= $iBound Next Local $a1D[$iTemp], $iIndex1D = 0 For $i = 0 To $aiBound[1] For $j = 0 To $aiBound[2] For $k = 0 To $aiBound[3] For $l = 0 To $aiBound[4] For $m = 0 To $aiBound[5] For $n = 0 To $aiBound[6] For $o = 0 To $aiBound[7] For $p = 0 To $aiBound[8] For $q = 0 To $aiBound[9] For $r = 0 To $aiBound[10] For $s = 0 To $aiBound[11] $a1D[$iIndex1D] = $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r][$s] $iIndex1D += 1 Next Next Next Next Next Next Next Next Next Next Next _ArrayShuffle($a1D) $iIndex1D = 0 For $i = 0 To $aiBound[1] For $j = 0 To $aiBound[2] For $k = 0 To $aiBound[3] For $l = 0 To $aiBound[4] For $m = 0 To $aiBound[5] For $n = 0 To $aiBound[6] For $o = 0 To $aiBound[7] For $p = 0 To $aiBound[8] For $q = 0 To $aiBound[9] For $r = 0 To $aiBound[10] For $s = 0 To $aiBound[11] $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r][$s] = $a1D[$iIndex1D] $iIndex1D += 1 Next Next Next Next Next Next Next Next Next Next Next EndFunc ; Conclusion: there's something wrong with my method! Thanks for taking the time to test it. Edited March 13, 2015 by czardas operator64Â Â ArrayWorkshop
JohnOne Posted March 13, 2015 Posted March 13, 2015 Oof! 2D array is one thing, MD is a different barrel of crabs AutoIt Absolute Beginners  Require a serial  Pause Script  Video Tutorials by Morthawt  ipify Monkey's are, like, natures humans.
czardas Posted March 13, 2015 Author Posted March 13, 2015 (edited) 2D array is one thing, MD is a different barrel of crabs  Indeed, but the method used in the code you posted is preferable to mine. I also see what is happening now I've had a good night's rest. It may not be difficult to change to your method. I was never entirely happy with my original code. Edit: It's less straight forward to modify than I thought. Edited March 13, 2015 by czardas operator64  ArrayWorkshop
czardas Posted March 13, 2015 Author Posted March 13, 2015 (edited) Unless I can figure out a better way to assign values to arrays with an unspecified number of dimensions, this method of optimization will introduce bloat. I remember having concerns about this before. Either you allow some latency by checking the number of dimensions every time you write to the array, or you hard code the loops for each separate case as shown below. Incidentally I reduced the number of dimensions to 10 and ran a comparison between this and the original function. It's an improvement in terms of speed, but if you know in advance that your array will be two or three dimensions (etc) then a taylor-made function will always have the edge. ; expandcollapse popup#include <array.au3> Local $aTest[15][15] For $i = 0 To 9 For $j = 0 To 9 $aTest[$i][$j] = $i & $j Next Next _ArrayDisplay($aTest) $iLoops = 500 $iTimer = TimerInit() For $i = 0 To $iLoops _ArrayShuffleMultiDim($aTest) Next ConsoleWrite("_ArrayShuffleMultiDim " & TimerDiff($iTimer) & @LF) ;_ArrayDisplay($aTest) $ITimer = TimerInit() For $i = 0 To $iLoops _ArrayShuffleMultiDim_ORIGINAL($aTest) Next ConsoleWrite("_ArrayShuffleMultiDim_ORIGINAL " & TimerDiff($iTimer) & @LF) ;_ArrayDisplay($aTest) Func _ArrayShuffleMultiDim(ByRef $aArray) If IsArray($aArray) = 0 Then Return SetError (1) Local $aiBound[11] = [UBound($aArray, 0),0,0,0,0,0,0,0,0,0,0] If $aiBound[0] = 0 Or $aiBound[0] > 10 Then Return SetError (2) ; only supports up to 10 dimensions Local $iBound = 1 For $i = 1 To $aiBound[0] ; for each dimension $aiBound[$i] = UBound($aArray, $i) ; get dimension size $iBound *= $aiBound[$i] $aiBound[$i] -= 1 Next Local $a1D[$iBound], $sSyntax = StringLeft("[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r]", $aiBound[0] *4), _ $iCount = 0 For $r = 0 To $aiBound[10] For $q = 0 To $aiBound[9] For $p = 0 To $aiBound[8] For $o = 0 To $aiBound[7] For $n = 0 To $aiBound[6] For $m = 0 To $aiBound[5] For $l = 0 To $aiBound[4] For $k = 0 To $aiBound[3] For $j = 0 To $aiBound[2] For $i = 0 To $aiBound[1] $a1D[$iCount] = Execute("$aArray" & $sSyntax) $iCount += 1 Next Next Next Next Next Next Next Next Next Next _ArrayShuffle($a1D) $iCount = 0 Switch $aiBound[0] ; number of dimensions Case 1 $aArray = $a1D Case 2 For $i = 0 To $aiBound[1] For $j = 0 To $aiBound[2] $aArray[$i][$j] = $a1D[$iCount] $iCount += 1 Next Next Case 3 For $i = 0 To $aiBound[1] For $j = 0 To $aiBound[2] For $k = 0 To $aiBound[3] $aArray[$i][$j][$k] = $a1D[$iCount] $iCount += 1 Next Next Next Case 4 For $i = 0 To $aiBound[1] For $j = 0 To $aiBound[2] For $k = 0 To $aiBound[3] For $l = 0 To $aiBound[4] $aArray[$i][$j][$k][$l] = $a1D[$iCount] $iCount += 1 Next Next Next Next Case 5 For $i = 0 To $aiBound[1] For $j = 0 To $aiBound[2] For $k = 0 To $aiBound[3] For $l = 0 To $aiBound[4] For $m = 0 To $aiBound[5] $aArray[$i][$j][$k][$l][$m] = $a1D[$iCount] $iCount += 1 Next Next Next Next Next Case 6 For $i = 0 To $aiBound[1] For $j = 0 To $aiBound[2] For $k = 0 To $aiBound[3] For $l = 0 To $aiBound[4] For $m = 0 To $aiBound[5] For $n = 0 To $aiBound[6] $aArray[$i][$j][$k][$l][$m][$n] = $a1D[$iCount] $iCount += 1 Next Next Next Next Next Next Case 7 For $i = 0 To $aiBound[1] For $j = 0 To $aiBound[2] For $k = 0 To $aiBound[3] For $l = 0 To $aiBound[4] For $m = 0 To $aiBound[5] For $n = 0 To $aiBound[6] For $o = 0 To $aiBound[7] $aArray[$i][$j][$k][$l][$m][$n][$o] = $a1D[$iCount] $iCount += 1 Next Next Next Next Next Next Next Case 8 For $i = 0 To $aiBound[1] For $j = 0 To $aiBound[2] For $k = 0 To $aiBound[3] For $l = 0 To $aiBound[4] For $m = 0 To $aiBound[5] For $n = 0 To $aiBound[6] For $o = 0 To $aiBound[7] For $p = 0 To $aiBound[8] $aArray[$i][$j][$k][$l][$m][$n][$o][$p] = $a1D[$iCount] $iCount += 1 Next Next Next Next Next Next Next Next Case 9 For $i = 0 To $aiBound[1] For $j = 0 To $aiBound[2] For $k = 0 To $aiBound[3] For $l = 0 To $aiBound[4] For $m = 0 To $aiBound[5] For $n = 0 To $aiBound[6] For $o = 0 To $aiBound[7] For $p = 0 To $aiBound[8] For $q = 0 To $aiBound[9] $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q] = $a1D[$iCount] $iCount += 1 Next Next Next Next Next Next Next Next Next Case 10 For $i = 0 To $aiBound[1] For $j = 0 To $aiBound[2] For $k = 0 To $aiBound[3] For $l = 0 To $aiBound[4] For $m = 0 To $aiBound[5] For $n = 0 To $aiBound[6] For $o = 0 To $aiBound[7] For $p = 0 To $aiBound[8] For $q = 0 To $aiBound[9] For $r = 0 To $aiBound[10] $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r] = $a1D[$iCount] $iCount += 1 Next Next Next Next Next Next Next Next Next Next EndSwitch EndFunc ;==> _ArrayShuffleMultiDim Func _ArrayShuffleMultiDim_ORIGINAL(ByRef $aArray) If IsArray($aArray) = 0 Then Return SetError (1) Local $iDim = UBound($aArray, 0) ; Get No dimensions If $iDim > 11 Then Return SetError (2) Local $a[$iDim +1], $aiBound[12] = [0,0,0,0,0,0,0,0,0,0,0,0] For $i = 1 To $iDim ; For each dimensionn $aiBound[$i] = UBound($aArray, $i) -1 ; Get dimension size Next For $i = 0 To $aiBound[1] For $j = 0 To $aiBound[2] For $k = 0 To $aiBound[3] For $l = 0 To $aiBound[4] For $m = 0 To $aiBound[5] For $n = 0 To $aiBound[6] For $o = 0 To $aiBound[7] For $p = 0 To $aiBound[8] For $q = 0 To $aiBound[9] For $r = 0 To $aiBound[10] For $s = 0 To $aiBound[11] For $z = 1 To $iDim ; Get random values for each dimension. $a[$z] = Random(0, $aiBound[$z], 1) Next __Swap($aArray, $iDim, $a, $i, $j, $k, $l, $m, $n, $o, $p, $q, $r, $s) Next Next Next Next Next Next Next Next Next Next Next EndFunc Func __Swap(ByRef $aArray, $iDim, $a, $i, $j, $k, $l, $m, $n, $o, $p, $q, $r, $s) Local $vTemp Switch $iDim Case 1 $vTemp = $aArray[$i] $aArray[$i] = $aArray[$a[1]] $aArray[$a[1]] = $vTemp Case 2 $vTemp = $aArray[$i][$j] $aArray[$i][$j] = $aArray[$a[1]][$a[2]] $aArray[$a[1]][$a[2]] = $vTemp Case 3 $vTemp = $aArray[$i][$j][$k] $aArray[$i][$j][$k] = $aArray[$a[1]][$a[2]][$a[3]] $aArray[$a[1]][$a[2]][$a[3]] = $vTemp Case 4 $vTemp = $aArray[$i][$j][$k][$l] $aArray[$i][$j][$k][$l] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]] = $vTemp Case 5 $vTemp = $aArray[$i][$j][$k][$l][$m] $aArray[$i][$j][$k][$l][$m] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]] = $vTemp Case 6 $vTemp = $aArray[$i][$j][$k][$l][$m][$n] $aArray[$i][$j][$k][$l][$m][$n] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]] = $vTemp Case 7 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o] $aArray[$i][$j][$k][$l][$m][$n][$o] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]] = $vTemp Case 8 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o][$p] $aArray[$i][$j][$k][$l][$m][$n][$o][$p] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]] = $vTemp Case 9 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q] $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]] = $vTemp Case 10 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r] $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]][$a[10]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]][$a[10]] = $vTemp Case 11 $vTemp = $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r][$s] $aArray[$i][$j][$k][$l][$m][$n][$o][$p][$q][$r][$s] = $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]][$a[10]][$a[11]] $aArray[$a[1]][$a[2]][$a[3]][$a[4]][$a[5]][$a[6]][$a[7]][$a[8]][$a[9]][$a[10]][$a[11]] = $vTemp EndSwitch EndFunc Edited March 13, 2015 by czardas operator64Â Â ArrayWorkshop
Recommended Posts
Create an account or sign in to comment
You need to be a member in order to leave a comment
Create an account
Sign up for a new account in our community. It's easy!
Register a new accountSign in
Already have an account? Sign in here.
Sign In Now