Break web sourcecode into chunks to get to desired data. UDF

[Morthawt]

expandcollapse popup

#include-once
#include <String.au3>

#cs
    ;   To program the function with before and after areas do the following
    (This breaks the source code down into chunks, then breaks those chunks into chunks and then breaks those chunks into chunks until you arrive at the exact data you need.
    By the way this also includes multiple entries of data, this following example shows one piece of data only and thus will return 1 entry array. More data sections = more array entries returned.

    For example Source code containing

    "<BeforeMain>This source code is big<CloserMain>This right here is not the current temperature outside<RightBetweenHere>GOLD FINGER<AndThisPart> But one day it could be<AfterCloserMain>But not big enough to cause a problem<AfterMain>"

    Chunk(0, '<BeforeMain>', '<AfterMain>')
    Chunk(0, '<CloserMain>', '<AfterCloserMain>')
    Chunk(0, '<RightBetweenHere>', '<AndThisPart>')

    ;   Then to fire it off to get a return array containing all of the resulting chunks of data

    $Variable = Chunk('http://www.google.com')

    ;   Or you can provide raw source code in binary format to prevent online retrieval of source code that you may be getting multiple different pieces of data from one piece of source.

    $Variable = Chunk($WebPageBinarySourceCode)

    ;   Or to remove all leading/trailing spaces, tabs, new lines etc...

    $Variable = Chunk('http://www.google.com', 1)

    ; Or to run the processing case sensitive add an extra parameter after the space stripping one shown above...

    $Variable = Chunk('http://www.google.com', 1, 1) ; Case sensitive

    $Variable = Chunk('http://www.google.com', 1, 0) ; Not case sensitive

    $Variable = Chunk('http://www.google.com', 1) ; Also not case sensitive due to the parameter being missing.

    ;   To clear all previously input chunk parameters you can do the following:

    Chunk(1)
#ce

Func Chunk($_BinarySourceOrStringURL, $_Import1 = '', $_Import2 = '')
    Local Static $_Zones[0]
    Select
        Case $_BinarySourceOrStringURL = String(0)
            If Not $_Import1 Or Not $_Import2 Then
                Local $_error[1]
                SetError(3) ; Both chunk before and after parameters are required for chunk parameters to be entered. All chunk parameters will now be reset. Please fix your script to supply both chunk before and after parameters.
                ReDim $_Zones[0]
                Return $_error
            EndIf
            ReDim $_Zones[UBound($_Zones) + 2]
            $_Zones[UBound($_Zones) - 2] = $_Import1
            $_Zones[UBound($_Zones) - 1] = $_Import2
            Return
        Case $_BinarySourceOrStringURL = String(1)
            ReDim $_Zones[0]
    EndSelect
    Local $_chunks[0]
    Select
        Case UBound($_Zones) = 0
            Local $_error[1]
            SetError(4) ; Both chunk before and after parameters are required. Either you forgot to add one or called the function to process source/URL without setting chunk parameters.
            ReDim $_Zones[0]
            Return $_error
        Case IsBinary($_BinarySourceOrStringURL)
            $_BinarySourceOrStringURL = BinaryToString($_BinarySourceOrStringURL)
        Case IsString($_BinarySourceOrStringURL)
            $_source = InetRead($_BinarySourceOrStringURL, 3)
            If @error Then
                Local $_error[1]
                Select
                    Case Not $_BinarySourceOrStringURL
                        SetError(1) ; Binary source code should have been provided but wasn't. The URL that was used is incorrect or there's a network problem. You may try again, the chunk parameters are intact.
                    Case $_BinarySourceOrStringURL
                        SetError(2) ; A URL was provided but there was a problem accessing it. Likely a network issue or a bad URL. You may try again, the chunk parameters are intact.
                EndSelect
                ReDim $_Zones[0]
                Return $_error
            EndIf
            $_BinarySourceOrStringURL = BinaryToString($_source)
    EndSelect
    Switch $_Import2
        Case 0
            $_CaseSenseChoice = False
        Case 1
            $_CaseSenseChoice = True
    EndSwitch
    ReDim $_chunks[UBound($_chunks) + 1]
    $_ProcessThis = _BetweenList($_BinarySourceOrStringURL, $_Zones[0], $_Zones[1], $_CaseSenseChoice)
    If @error Then
        SetError(@error)
        Local $_error[1]
        ReDim $_Zones[0]
        Return $_error
    EndIf
    For $Go = 2 To (UBound($_Zones) - 1) Step 2
        $_ProcessThis = _BetweenList($_ProcessThis, $_Zones[$Go], $_Zones[$Go + 1], $_CaseSenseChoice)
        If @error Then
            SetError(@error)
            Local $_error[1]
            ReDim $_Zones[0]
            Return $_error
        EndIf
    Next
    If $_Import1 Then
        Local $_TempBuffer[0]
        For $ap In $_ProcessThis
            ReDim $_TempBuffer[UBound($_TempBuffer) + 1]
            $_TempBuffer[UBound($_TempBuffer) - 1] = StringStripWS(StringStripCR(StringReplace(StringReplace($ap, @TAB, ''), @LF, '')), 3)
        Next
        ReDim $_Zones[0]
        Return $_TempBuffer
    EndIf
    ReDim $_Zones[0]
    Return $_ProcessThis
EndFunc   ;==>Chunk

Func _BetweenList($_DataArrayInput, $_Before = '', $_After = '', $_CaseSensitive = False)
    Local $_Buffer[0]
    If Not IsArray($_DataArrayInput) Then
        $_tempCheck = _StringBetween($_DataArrayInput, $_Before, $_After, $_CaseSensitive)
        If Not @error Then
            Return $_tempCheck
        Else
            SetError(5) ; Nothing came back from the breakdown of the source.
            Local $_error[1]
            Return $_error
        EndIf
    EndIf
    For $_Process In $_DataArrayInput
        $_temp = _StringBetween($_Process, $_Before, $_After, $_CaseSensitive)
        If Not @error Then
            For $_processing In $_temp
                ReDim $_Buffer[UBound($_Buffer) + 1]
                $_Buffer[UBound($_Buffer) - 1] = $_processing
            Next
        EndIf
    Next

    If Not UBound($_Buffer) Then
        SetError(6) ; Nothing came back from the subsequent breakdowns of the source.
        Local $_error[1]
        Return $_error
    EndIf
    Return $_Buffer
EndFunc   ;==>_BetweenList

 

Above is the UDF I made for myself that I am sharing.

Changes: 11th Jan 2014

The error information was not being provided to the user because they were mainly being produced by the internal function used by the user's called function. Now I have fixed that so that a user will have an error code reproduced to them so they can identify what caused the error. Additional new fix for today: Fixed error tracking to prevent false errors, meaning now you can supply an array to for the function to filter through for you and it will process it by going through each entry in the array searching for your data as you would expect. So now it can handle string URL, binary source code or array containing string data.

Changes 12th Jan 2014

Fixed an issue where the programmed "static" chunk zones were not being reset after returning from an error.

 

There are examples in the top comments area of the UDF, but I will include a couple of examples below...

; This example splits the truecrypt.org website up to retrieve the truecrypt versions from the news area on the left of the page.
; Case insensitive search without stripping any spaces, new lines, tabs etc because there are none.
#include <Includes\web-breakdown.au3>

$Source = InetRead('http://www.truecrypt.org', 3)

WebPageBreakDown(0, '<span class="smallFnt"><b>News</b></span>', '</span></td>')
WebPageBreakDown(0, '<a href="', '</span>')
WebPageBreakDown(0, '">', '<Br>')

$Result = WebPageBreakDown($Source, 0, 0)
If Not @error Then
    For $a In $Result
        ConsoleWrite($a & @CRLF)
    Next
Else
    If @error Then ConsoleWrite('Error code: ' & @error & @CRLF)
EndIf

; This example splits the truecrypt.org website up to retrieve the truecrypt versions from the news area on the left of the page.
; Case [sensitive] search without stripping any spaces, new lines, tabs etc because there are none.
; Returns no results due to the case being incorrect for the "after" chunk parameter of the third set of chunks <Br> should be <br> for case sensitive to retrieve the data.
#include <Includes\web-breakdown.au3>

$Source = InetRead('http://www.truecrypt.org', 3)

WebPageBreakDown(0, '<span class="smallFnt"><b>News</b></span>', '</span></td>')
WebPageBreakDown(0, '<a href="', '</span>')
WebPageBreakDown(0, '">', '<Br>')

$Result = WebPageBreakDown($Source, 0, 1)
If Not @error Then
    For $a In $Result
        ConsoleWrite($a & @CRLF)
    Next
Else
    If @error Then ConsoleWrite('Error code: ' & @error & @CRLF)
EndIf

If you have any feedback on this I would be interested in it. I made this for myself but I have decided that I will share it in case any other people can make use of it. I tend to make little programs to check a website to let me know when a new [something] shows up and rather than code it all from scratch each time (which is a pain due to processing array entries and then each entry to new array entries etc) I decided to come up with a way that I could make a UDF to make it easier in future.

So this function is multi-purpose because you use the same function to both prime the chunk parameters to get closer and closer to the data until only the data you need are between the last before/after chunks and you also use the same function to initiate the processing of the source code to retrieve an array containing all the data you want.

Each time you finally process the source code after having input the chunk parameters, it will automatically reset previous chunk parameters so the next time you run the function you will need to prime it with new chunks. You can also manually reset the chunk parameters for some reason by using one parameter of 1. It is all detailed on the comments area of the UDF its self.

I have added as much protection as I can to protect against crashes, added error levels and comments on them so you know what an error is for.

Let me know what you think.

Morthawt

Edited January 12, 2014 by Morthawt

[wakillon]

Hum...place a _StringBetween  after a return is not a good idea 

Instead, use a variable and check if no error would be more secure 

[Morthawt]

Where are you referring to? I have multiple returns based on different events taking place.

[wakillon]

Where are you referring to? I have multiple returns based on different events taking place.

 

This line : If Not IsArray($_DataArrayInput) Then Return _StringBetween($_DataArrayInput, $_Before, $_After, $_CaseSensitive)

[Morthawt]

Should be fixed, had some issues trying to edit the code it looks a mess now.. I just saw your reply. I figured that was what you were referring to. I edited the source, how can I get it to look good again on the first post? I will keep trying.

EDIT: Got it.. done. Took some messing around to get it not to sprawl all over the page outside the autoit code area.

Edited January 7, 2014 by Morthawt

[wakillon]

Should be fixed, had some issues trying to edit the code it looks a mess now.. I just saw your reply. I figured that was what you were referring to. I edited the source, how can I get it to look good again on the first post? I will keep trying.

EDIT: Got it.. done. Took some messing around to get it not to sprawl all over the page outside the autoit code area.

 

Why did you say it do not looks good ?

Problem for edit your post ?

[Morthawt]

Yeah, it went all screwy but I fixed it. I fixed the thing you pointed out, it was actually the same thing you were referring to. This thing can be used for things other than websites, just convert the data to Binary before you feed it in.

Do you have any comments on this UDF's functionality? Think it will be useful to others?

[guinness]

Create a standard header for documentation, it's useful as it's a familiar format for most people.

[Morthawt]

Create a standard header for documentation, it's useful as it's a familiar format for most people.

 

I was experimenting with it but I was having an issue with the formatting of it trying to incorporate all my examples and notes. I pretty much just released it as is in the end, with my own details added.

[guinness]

Ctrl+Alt+H on the function line and voila. 

Edited January 7, 2014 by guinness

[Morthawt]

Yes I had messed around with it but I was having issues trying to get everything typed and what not. I ended up ditching it and writing my own. It is only there to explain how to make use of it. I decided to add a comment area than just write it on the thread.

EDIT

I checked again, it is useless because my parameters are multi-purpose so trying to write it all out like that is confusing because the parameters are not for a single input type/purpose. I made it easy to use but it requires explanation not a simple this = that and requires multiple examples to show in which context to put what type of content in the parameters.

Edited January 7, 2014 by Morthawt

[guinness]

Hmm, I think you shouldn't call this a UDF in that case.

[Morthawt]

It is not a built in function, it is a function that is user defined. I am not lobbying to get it included with Autoit, something which would require everything to be 100% standards complient. I am just sharing something I made for myself that other people can make use of if they want to. It is working very well for me.

[Morthawt]

Here is an example I just made (I renamed the function to Chunk for ease and speed) that checks theregister.co.uk newsfeed for news and notifies me of anything new showing up:

expandcollapse popup

#include <Includes\web-breakdown.au3>
#include <Array.au3>
AutoItSetOption('TrayAutoPause', 0)

$Source = InetRead('http://www.theregister.co.uk/headlines.atom', 3)
$OriginalURLs = URLs()

While 1
    $Source = InetRead('http://www.theregister.co.uk/headlines.atom', 3)
    $TmpTitles = Titles()
    $TmpURLs = URLs()
    Local $New = ''

    For $a = 0 To UBound($TmpURLs) - 1
        If _ArraySearch($OriginalURLs, $TmpURLs[$a]) = -1 Then
            ReDim $OriginalURLs[UBound($OriginalURLs) + 1]
            $OriginalURLs[UBound($OriginalURLs) - 1] = $TmpURLs[$a]
            If UBound($TmpTitles) >= $a Then $New &= $TmpTitles[$a] & @CRLF & @CRLF
        EndIf
    Next
    If $New <> '' Then
        MsgBox(0, 'New news stories', $New, 30)
    EndIf

    Sleep(30000)
WEnd

Func Titles()
    Chunk(0, '<entry>', '</entry>')
    Chunk(0, '<title type="html">', '</title>')
    Return Chunk($Source)
EndFunc   ;==>Titles

Func URLs()
    Chunk(0, '<entry>', '</entry>')
    Chunk(0, '<link rel=', '/>')
    Chunk(0, 'href="', '"')
    Return Chunk($Source)
EndFunc   ;==>URLs

[Morthawt]

Updated the sourcecode.

The error information and even the existence of an error was not being translated to the user's call of the function. The errors were internal to the internal function used by the function the user calls. So I have added script that will provide a new SetError to the user's call of the function. I have also renamed the function used by the user to "Chunk" for speed and ease.

EDIT, fixed another issue, details on first post.

Edited January 11, 2014 by Morthawt

[Morthawt]

Updated. Details in first post.