#include <WindowsConstants.au3>
#include <Array.au3>
#include <String.au3>
#include <Date.au3>
#include <FileConstants.au3>
#include <WinAPIFiles.au3>
#include <File.au3>
#include <Math.au3>
#include <StaticConstants.au3>
#Include <Memory.au3>

Global $mTic=100,$mData=24000,$nData,$nTicker,$ticker[$mTic]
Global $last[$mTic][$mData],$bid[$mTic][$mData],$ask[$mTic][$mData],$vol[$mTic][$mData]
Global $time[$mData],$timeDec[$mData],$elapsed[$mData]
Global $onTicker[$mTic]

Local $directory = @DesktopDir & "\Project\data\"

;read ascii file
$timer = TimerInit()
readASCII($directory & "2019_02_08.csv")
ConsoleWrite("Ascii read & process time: " & TimerDiff($timer) & @LF)

;correct time errors
;fixTime()

;write binary file
$timer = TimerInit()
writeBinary($directory & "2019_02_08.bin")
ConsoleWrite("Binary write file time: " & TimerDiff($timer) & @LF)

;0 some data so we know it was read correctly in the check below
;For $i = 0 To 10
;	$ticker[$i] = ""
;	For $j = 0 To 10
;		$last[$i][$j] = 0.0
;		$bid[$i][$j] = 0.0
;		$ask[$i][$j] = 0.0
;		$vol[$i][$j] = 0
;	Next
;Next

;read in binary file
$timer = TimerInit()
readBinary($directory & "2019_02_08.bin")
ConsoleWrite("Binary read and process time: " & TimerDiff($timer) & @LF)

;display some results
;For $i = 0 To 10
;	consoleWrite($ticker[$i] & @CRLF)
;	For $j = 0 To 10
;		consoleWrite($last[$i][$j] & ", " & $bid[$i][$j] & ", " & $ask[$i][$j] & ", " & $vol[$i][$j] & @CRLF)
;	Next
;Next


;////readASCII////////////////////////////////////////////////////////
Func readASCII($fileName)
	$timer1 = TimerInit()
	$file = FileOpen($fileName,$FO_READ)
	If $file = -1 Then
		msgBox(262144,"Error","Error: Cannot open input file: " & @CRLF & $fileName)
		Exit
	EndIf
	$buffer = FileRead($file)
	FileClose($file)
	ConsoleWrite("Time to read ascii file only: " & TimerDiff($timer1) & @LF)

    $bufferLine = StringSplit($buffer,@CRLF,1)
	If UBound($bufferLine) < 3 Then
		msgBox(262144,"Error","Error: Input file contains no data: " & @CRLF & $fileName)
		Exit
	EndIf
	For $i = 1 To $bufferLine[0]
		$lineSplit = StringSplit($bufferLine[$i],",",1)
		If $i = 1 Then
			$nTicker = 0

			For $j = 2 To $lineSplit[0]-5 Step 5
				$ticker[$nTicker] = $lineSplit[$j]
				$onTicker[$nTicker] = False
				$nTicker += 1
			Next
			If ($nTicker > $mTic) Then
				msgBox(262144,"Error","Error: The array size for number of ticker symbols was exceeded: " & @CRLF)
				Exit
			EndIf
			$nData = 0
		ElseIf (Number($nTicker)*5)+2 <> $lineSplit[0] AND $lineSplit[0] > 1 Then
			msgBox(262144,"Error","Error: The number of tickers has changed in file: " & @CRLF & $fileName)
			Exit
		EndIf

		If $lineSplit[0] > 1 Then
			$time[$nData] = $lineSplit[1]
			$timeTemp = $lineSplit[1]
			$timeTempSplit = StringSplit($timeTemp,":")
			$timeTempSplit[3] = StringReplace($timeTempSplit[3]," AM","")
			$timeTempSplit[3] = StringReplace($timeTempSplit[3]," PM","")
			$hourTime = Number($timeTempSplit[1],3)
			$minTime = Number($timeTempSplit[2],3)
			$secTime = Number($timeTempSplit[3],3)
			$timeDec[$nData] = $hourTime + ((($minTime*60)+$secTime)/3600)
			$elapsed[$nData] = Number($lineSplit[$lineSplit[0]],3)
			$nTempTicker = 0
			$keepVals = True
			For $j = 3 To $lineSplit[0]-4 Step 5
				$last[$nTempTicker][$nData] = Number($lineSplit[$j],3)
				$bid[$nTempTicker][$nData] = Number($lineSplit[$j+1],3)
				$ask[$nTempTicker][$nData] = Number($lineSplit[$j+2],3)
				If $last[$nTempTicker][$nData] < 0.0 Then ;<= 0.0 Then
					$keepVals = False
				EndIf
				$vol[$nTempTicker][$nData] = Number($lineSplit[$j+3],2)
				$nTempTicker += 1
			Next
			If $keepVals Then
				$nData += 1
				If ($nData > $mData) Then
					msgBox(262144,"Error","Error: The array size for price data was exceeded: " & @CRLF)
					Exit
				EndIf
			EndIf
		EndIf
	Next

	;now set all values before first change in last to equal the first changed value
	For $i = 0 To $nTicker-1
		For $j = 1 To $nData-1
			if $last[$i][$j] <> $last[$i][0] Then
				ExitLoop
			EndIf
		Next
		For $k = 0 To $j-1
			$last[$i][$k] = $last[$i][$j]
		Next
		For $j = 1 To $nData-1
			If $bid[$i][$j] = 0.0 Then
				$bid[$i][$j] = $last[$i][$j]
			EndIf
			If $ask[$i][$j] = 0.0 Then
				$ask[$i][$j] = $last[$i][$j]
			EndIf
		Next
	Next
;_arrayDisplay($timeDec)
;_arrayDisplay($elapsed)
EndFunc


;////writeBinary////////////////////////////////////////////////////////
Func writeBinary($fileName)
    $file = FileOpen($fileName,$FO_OVERWRITE + $FO_BINARY)

	;write size of arrays
	$tickerNameLen = 100	;number of chars to store for ticker symbol name
	writeBinaryInt32($file,$nTicker)  ;number of ticker symbols
	writeBinaryInt32($file,$nData)    ;number of data points for each symbol

	;write ticker symbols
	writeBinaryInt32($file,$tickerNameLen)	;length of string for ticker symbol
	For $i = 0 To $nTicker-1
		writeBinaryString($file,$ticker[$i],$tickerNameLen)		;write ticker symbol name
	Next

	;write time of day as decimal: 9:30am = 9.5
	For $i = 0 to $nData-1
		writeBinaryDouble($file,$timeDec[$i])
	Next

	;write elapsed time needed to get data from server
	For $i = 0 to $nData-1
		writeBinaryDouble($file,$elapsed[$i])
	Next

	;write last, bid, ask, vol
	For $i = 0 to $nTicker-1
		For $j = 0 to $nData-1
			writeBinaryDouble($file,$last[$i][$j])
		Next
	Next
	For $i = 0 to $nTicker-1
		For $j = 0 to $nData-1
			writeBinaryDouble($file,$bid[$i][$j])
		Next
	Next
	For $i = 0 to $nTicker-1
		For $j = 0 to $nData-1
			writeBinaryDouble($file,$ask[$i][$j])
		Next
	Next
	For $i = 0 to $nTicker-1
		For $j = 0 to $nData-1
			writeBinaryInt64($file,$vol[$i][$j])
		Next
	Next

	FileClose($file)
EndFunc


;////readBinary////////////////////////////////////////////////////////
Func readBinary($fileName)
	;read binary file
	$timer1 = TimerInit()
	$file = FileOpen($fileName,$FO_READ + $FO_BINARY)
	Local $data = fileRead($file)
	FileClose($file)
	ConsoleWrite("Time just to read binary file: " & TimerDiff($timer1) & @LF)

	;put binary array size data into struct
	$tSizes = DllStructCreate( "int[" & 3 & "]" )
	$pSizes = DllStructGetPtr( $tSizes )
	$tBytes =  DllStructCreate( "byte[" & 3*4 & "]", $pSizes )
	DLLStructSetData($tBytes, 1, $data)

	;copy struct data into variables
	$nTicker = DLLStructGetData($tSizes, 1, 1)
	$nData = DLLStructGetData($tSizes, 1, 2)
	$tickerNameLen = DLLStructGetData($tSizes, 1, 3)
	$tSizes = 0

	;put binary ticker names into struct
	$tNames = DllStructCreate( "char[" & $tickerNameLen & "]" )
	$pNames = DllStructGetPtr( $tNames )
	$tBNames = DLLStructCreate( "byte[" & $tickerNameLen & "]", $pNames )

	;copy struct data into variables
	For $i = 0 To $nTicker-1
		DLLStructSetData($tBNames, 1, BinaryMid($data, 13+($i*$tickerNameLen)))
		$ticker[$i] = BinaryToString(DLLStructGetData($tBNames, 1))
	Next
	$tNames = 0

	;put binary time and price data into struct
	$tData = DllStructCreate( "double[" & $nData & "];double[" & $nData & "];double[" & $nTicker*$nData & "];double[" & $nTicker*$nData & "];double[" & $nTicker*$nData & "];uint64[" & $nTicker*$nData & "]")
	$pData = DllStructGetPtr( $tData )
	$tDBytes =  DllStructCreate( "byte[" & (2*$nData*8) + (4*$nTicker*$nData*8) & "]", $pData )
	DLLStructSetData($tDBytes, 1, BinaryMid($data, 13+($nTicker*$tickerNameLen)))

	;copy struct data into variables
	For $i = 0 to $nData-1
		$timeDec[$i] = DLLStructGetData($tData, 1, $i+1)
		$elapsed[$i] = DLLStructGetData($tData, 2, $i+1)
	Next
	For $i = 0 To $nTicker-1
		For $j = 0 To $nData-1
			$byteLoc = 1 + ($i*$nData) + $j
			$last[$i][$j] = DLLStructGetData($tData, 3, $byteLoc)
			$bid[$i][$j] = DLLStructGetData($tData, 4, $byteLoc)
			$ask[$i][$j] = DLLStructGetData($tData, 5, $byteLoc)
			$vol[$i][$j] = DLLStructGetData($tData, 6, $byteLoc)
		Next
	Next
	$tData = 0
EndFunc


;////writeBinaryInt32////////////////////////////////////////////////////////
Func writeBinaryInt32($filePointer,$intVal)
	fileWrite($filePointer,Binary(Number($intVal,1)))
EndFunc


;////writeBinaryInt64////////////////////////////////////////////////////////
Func writeBinaryInt64($filePointer,$intVal)
	fileWrite($filePointer,Binary(Number($intVal,2)))
EndFunc


;////writeBinaryString////////////////////////////////////////////////////////
Func writeBinaryString($filePointer,$stringVal,$stringLen)
	$newStringVal = StringMid($stringVal,1,$stringLen) ;trims string that is too long
	While StringLen($newStringVal) < $stringLen
		$newStringVal = $newStringVal & " "   ;adds space to a string that is too short
	WEnd

	fileWrite($filePointer,StringToBinary($newStringVal))
EndFunc

;////writeBinaryDouble////////////////////////////////////////////////////////
Func writeBinaryDouble($filePointer,$doubleVal)
	fileWrite($filePointer,Binary(Number($doubleVal,3)))
EndFunc


;////binaryToInt32////////////////////////////////////////////////////////
Func binaryToInt32($file)
	Return _BinaryToInt32(FileRead($file, 4))
EndFunc


;////binaryToInt64////////////////////////////////////////////////////////
Func binaryToInt64($file)
	Return _BinaryToInt64(FileRead($file, 8))
EndFunc


;////binaryToStr////////////////////////////////////////////////////////
Func binaryToStr($file, $stringLen)
	Return BinaryToString(FileRead($file, $stringLen))
EndFunc


;////binaryToDouble////////////////////////////////////////////////////////
Func binaryToDouble($file)
	;Return Number(FileRead($file, 8),3)
     Return _BinaryToDouble(FileRead($file, 8))
EndFunc


;////fixTime////////////////////////////////////////////////////////
Func fixTime()
	$k = $nData-1
	For $i = 0 To $nTicker-1
		If Not StringInStr($ticker[$i],"$") Then
			$lastLast = $last[$i][0]
			For $j = 1 To $nData-1
				If $last[$i][$j] <> $lastLast Then
					If $j < $k Then
						$k = $j
					EndIf
					ExitLoop
				EndIf
			Next
		EndIf
	Next

	;consoleWrite("Time when last changes: " & $timeDec[$k] & @CRLF)
EndFunc


;Below are from: Binary UDF by "Ward"
Func _BinaryToInt32($Binary)
	Return BitAND(Int(Binary($Binary)), 0xFFFFFFFF)
EndFunc

Func _BinaryFromInt32($Value)
	Return Binary(BitAND($Value, 0xFFFFFFFF))
EndFunc

Func _BinaryToInt64($Binary)
	Local $Buffer = DllStructCreate("byte[8]")
	DllStructSetData($Buffer, 1, Binary($Binary))

	Return DllStructGetData(DllStructCreate("int64", DllStructGetPtr($Buffer)), 1)
EndFunc

Func _BinaryFromInt64($Value)
	Local $Buffer = DllStructCreate("int64")
	DllStructSetData($Buffer, 1, $Value)

	Return DllStructGetData(DllStructCreate("byte[8]", DllStructGetPtr($Buffer)), 1)
EndFunc

Func _BinaryToDouble($Binary)
	Local $Buffer = DllStructCreate("byte[8]")
	DllStructSetData($Buffer, 1, Binary($Binary))

	Return DllStructGetData(DllStructCreate("double", DllStructGetPtr($Buffer)), 1)
EndFunc

Func _BinaryFromDouble($Value)
	Local $Buffer = DllStructCreate("double")
	DllStructSetData($Buffer, 1, $Value)

	Return DllStructGetData(DllStructCreate("byte[8]", DllStructGetPtr($Buffer)), 1)
EndFunc