A Non-Strict JSON UDF (JSMN)

[emcodem]

Just to inform you about our research results, this library can cause a memory access violation error message that keeps windows from shutting down (stop shutdown and asking the user to hit OK).

Error msg: "The Instruction at 0x%address% referenced memory at 0x%same_address%. The memory could not be written. Click OK to terminate the program.

 
This happened only on certain Win10 machines and even there it was not always reproduceable, happening only e.g. 3 of 5 times.

So again, the error only occured when we used the library in the time window when windows is doing the shutdown, e.g. 
 

OnAutoItExitRegister("__exit_procedure") 

While 1
    Sleep(500)
WEnd

Func __exit_procedure()
    Json_Encode($object)
EndFunc
 

Run this program and then shutdown windows. 

screenshot here http://www.ffastrans.com/frm/forum/viewtopic.php?f=4&t=1184#p6154

 

Edited February 1, 2021 by emcodem

[Danp2]

@emcodem We ran into this issue with the Webdriver UDF as well. See discussion starting here for how to resolve.

[emcodem]

Thanks for the tipp @Danp2 ..if i am right, the workaround you applied was the same than we did: just don't execute the affected function in the OnAutoItExitRegister callback but somewhen else and grab the json encoded stuff from a global variable instead.

For example we worked around by just using regex on the already encoded json. But a workaround is not a fix and i fear we are not the only ones that suffer from such an issue and spend huge amounts of time to debug it. Thats why i thought its good to post this here.

What i can add additionally, it don't seem to be directly connected to OnAutoItExitRegister. We had the same behaviour in an older version of our program when detecting the program shutdown in our own code and were executing Json_Encode inside an AdlibRegister Function. Using this method, the Adlib function ran thousands of time at runtime without any problem but as it was executed in this time spot when windows shuts down, Json_encode caused the mentioned issue.

Unfortunately, below is not tested but this was about how our old program version worked hehe...

AdlibRegister("_ProcAdmin", 1000)
Func _ProcAdmin()
    ...lots of code
    Json_Encode($object)
    ...lots of code
EndFunc 

 

Edited February 1, 2021 by emcodem

[adityaparakh]

expandcollapse popup

{
    "type": "success",
    "code": "s-portfolio-0001",
    "description": "Success position",
    "result": {
        "positionList": [
            {
                "AccountID": "ASIA",
                "TradingSymbol": "BANKNIFTY 04FEB2021 CE 34700",
                "ExchangeSegment": "NSEFO",
                "ExchangeInstrumentId": "38544",
                "ProductType": "NRML",
                "Marketlot": "25",
                "Multiplier": "1",
                "BuyAveragePrice": "0.00",
                "SellAveragePrice": "394.89",
                "OpenBuyQuantity": "0",
                "OpenSellQuantity": "200",
                "Quantity": "-200",
                "BuyAmount": "0.00",
                "SellAmount": "78,977.50",
                "NetAmount": "78,977.50",
                "UnrealizedMTM": "34,447.50",
                "RealizedMTM": "0.00",
                "MTM": "34,447.50",
                "BEP": "394.89",
                "SumOfTradedQuantityAndPriceBuy": "0.00",
                "SumOfTradedQuantityAndPriceSell": "78,977.50",
                "MessageCode": 9002,
                "MessageVersion": 1,
                "TokenID": 0,
                "ApplicationType": 0,
                "SequenceNumber": 344181107247201
            },
            {
                "AccountID": "ASIA",
                "TradingSymbol": "BANKNIFTY 04FEB2021 PE 34100",
                "ExchangeSegment": "NSEFO",
                "ExchangeInstrumentId": "46488",
                "ProductType": "NRML",
                "Marketlot": "25",
                "Multiplier": "1",
                "BuyAveragePrice": "0.00",
                "SellAveragePrice": "401.79",
                "OpenBuyQuantity": "0",
                "OpenSellQuantity": "200",
                "Quantity": "-200",
                "BuyAmount": "0.00",
                "SellAmount": "80,358.75",
                "NetAmount": "80,358.75",
                "UnrealizedMTM": "-15,441.25",
                "RealizedMTM": "0.00",
                "MTM": "-15,441.25",
                "BEP": "401.79",
                "SumOfTradedQuantityAndPriceBuy": "0.00",
                "SumOfTradedQuantityAndPriceSell": "80,358.75",
                "MessageCode": 9002,
                "MessageVersion": 1,
                "TokenID": 0,
                "ApplicationType": 0,
                "SequenceNumber": 344181107247202
            },
            {
                "AccountID": "ASIA",
                "TradingSymbol": "BANKNIFTY 04FEB2021 PE 34000",
                "ExchangeSegment": "NSEFO",
                "ExchangeInstrumentId": "46486",
                "ProductType": "NRML",
                "Marketlot": "25",
                "Multiplier": "1",
                "BuyAveragePrice": "0.00",
                "SellAveragePrice": "356.82",
                "OpenBuyQuantity": "0",
                "OpenSellQuantity": "250",
                "Quantity": "-250",
                "BuyAmount": "0.00",
                "SellAmount": "89,203.75",
                "NetAmount": "89,203.75",
                "UnrealizedMTM": "-17,796.25",
                "RealizedMTM": "0.00",
                "MTM": "-17,796.25",
                "BEP": "356.82",
                "SumOfTradedQuantityAndPriceBuy": "0.00",
                "SumOfTradedQuantityAndPriceSell": "89,203.75",
                "MessageCode": 9002,
                "MessageVersion": 1,
                "TokenID": 0,
                "ApplicationType": 0,
                "SequenceNumber": 344181107247203
            },
            {
                "AccountID": "ASIA",
                "TradingSymbol": "BANKNIFTY 04FEB2021 PE 33900",
                "ExchangeSegment": "NSEFO",
                "ExchangeInstrumentId": "36195",
                "ProductType": "NRML",
                "Marketlot": "25",
                "Multiplier": "1",
                "BuyAveragePrice": "0.00",
                "SellAveragePrice": "315.74",
                "OpenBuyQuantity": "0",
                "OpenSellQuantity": "200",
                "Quantity": "-200",
                "BuyAmount": "0.00",
                "SellAmount": "63,147.50",
                "NetAmount": "63,147.50",
                "UnrealizedMTM": "-13,112.50",
                "RealizedMTM": "0.00",
                "MTM": "-13,112.50",
                "BEP": "315.74",
                "SumOfTradedQuantityAndPriceBuy": "0.00",
                "SumOfTradedQuantityAndPriceSell": "63,147.50",
                "MessageCode": 9002,
                "MessageVersion": 1,
                "TokenID": 0,
                "ApplicationType": 0,
                "SequenceNumber": 344181107247204
            },
            {
                "AccountID": "ASIA",
                "TradingSymbol": "BANKNIFTY 04FEB2021 CE 34600",
                "ExchangeSegment": "NSEFO",
                "ExchangeInstrumentId": "38540",
                "ProductType": "NRML",
                "Marketlot": "25",
                "Multiplier": "1",
                "BuyAveragePrice": "0.00",
                "SellAveragePrice": "430.22",
                "OpenBuyQuantity": "0",
                "OpenSellQuantity": "200",
                "Quantity": "-200",
                "BuyAmount": "0.00",
                "SellAmount": "86,043.75",
                "NetAmount": "86,043.75",
                "UnrealizedMTM": "36,433.75",
                "RealizedMTM": "0.00",
                "MTM": "36,433.75",
                "BEP": "430.22",
                "SumOfTradedQuantityAndPriceBuy": "0.00",
                "SumOfTradedQuantityAndPriceSell": "86,043.75",
                "MessageCode": 9002,
                "MessageVersion": 1,
                "TokenID": 0,
                "ApplicationType": 0,
                "SequenceNumber": 344181107247205
            },
            {
                "AccountID": "ASIA",
                "TradingSymbol": "BANKNIFTY 04FEB2021 CE 34500",
                "ExchangeSegment": "NSEFO",
                "ExchangeInstrumentId": "38537",
                "ProductType": "NRML",
                "Marketlot": "25",
                "Multiplier": "1",
                "BuyAveragePrice": "0.00",
                "SellAveragePrice": "469.17",
                "OpenBuyQuantity": "0",
                "OpenSellQuantity": "200",
                "Quantity": "-200",
                "BuyAmount": "0.00",
                "SellAmount": "93,833.75",
                "NetAmount": "93,833.75",
                "UnrealizedMTM": "38,503.75",
                "RealizedMTM": "0.00",
                "MTM": "38,503.75",
                "BEP": "469.17",
                "SumOfTradedQuantityAndPriceBuy": "0.00",
                "SumOfTradedQuantityAndPriceSell": "93,833.75",
                "MessageCode": 9002,
                "MessageVersion": 1,
                "TokenID": 0,
                "ApplicationType": 0,
                "SequenceNumber": 344181107247206
            }
        ]
    }
}

 

 

How can I get result.PositionList into a 2D array.
 

$jsontemp = Json_Decode($oReceived)
    $positionlist = Json_ObjGet($jsontemp, "result.positionList")

 

 

[TheXman]

On 2/2/2021 at 1:17 AM, adityaparakh said:

How can I get result.PositionList into a 2D array.
 

$jsontemp = Json_Decode($oReceived)
$positionlist = Json_ObjGet($jsontemp, "result.positionList")

 

@adityaparakh

Since you provided absolutely no information related to what your 2D array should look like or contain, the well-documented example below is very generic.  With 100's of relevant examples across the forums (including many in this topic alone), one would think you could have shown a little more effort than the two lines that you provided.

expandcollapse popup

#include <Constants.au3>
#include <Array.au3>
#include "json.au3" ; <== Modify as needed

Const $JSON_DATA = _
          '{' & _
          '   "type": "success",' & _
          '   "code": "123ABC",' & _
          '   "description": "Another simple example",' & _
          '   "result": {' & _
          '      "fruits": [' & _
          '         {"Fruit": "Apple"     , "Qty": 34700, "Cost": 94.89, "Status": false},' & _
          '         {"Fruit": "Banana"    , "Qty": 34100, "Cost": 01.79, "Status": true} ,' & _
          '         {"Fruit": "Orange"    , "Qty": 34000, "Cost": 56.82, "Status": true} ,' & _
          '         {"Fruit": "Mango"     , "Qty": 33900, "Cost": 15.74, "Status": false},' & _
          '         {"Fruit": "Pineapple" , "Qty": 34600, "Cost": 30.22, "Status": false},' & _
          '         {"Fruit": "Cantaloupe", "Qty": 34500, "Cost": 69.17, "Status": false}'  & _
          '      ]' & _
          '   }' & _
          '}'

json_udf_example()

Func json_udf_example()
    Local $oJson
    Local $aResult[0][3]

    ;Decode JSON to an object
    $oJson  = Json_Decode($JSON_DATA)
    If @error Then Exit MsgBox($MB_ICONERROR + $MB_TOPMOST, "ERROR", "Unable to decode JSON - @error = " & @error)

    ;For each object in the array
    For $oFruit in json_get($oJson, ".result.fruits")
        ;Add a row of data to the table
        _ArrayAdd($aResult, _
            Json_Get($oFruit, ".Fruit") & "|" & _
            Json_Get($oFruit, ".Qty")   & "|" & _
            Json_Get($oFruit, ".Cost") _
        )
        If @error Then Exit MsgBox($MB_ICONERROR + $MB_TOPMOST, "ERROR", "_ArrayAdd failed - @error = " & @error)
    Next

    ;Display result
    _ArrayDisplay($aResult, "Example 2D Array", "", 0, Default, "Fruit|Qty|Cost")
EndFunc

Edited February 6, 2021 by TheXman
Made script more generic

[TheDcoder]

Was the source code for the embedded binary/machine code ever released? I am interested in the encoding function as jsmn doesn't provide such a thing.

[TheXman]

3 hours ago, TheDcoder said:

I am interested in the encoding function as jsmn doesn't provide such a thing.

If you aren't able to find the original source, then at the bottom of this page there are numerous links to github JSON implementations in several different languages.  Or, you can always write your own JSON string encoding routine using RFC 7159 as your reference.

Edited February 10, 2021 by TheXman

[TheDcoder]

@TheXman Thank you for the links, I am aware of them. I did some research but I realized that I don't really need to depend on jsmn...

P.S For anyone who is interested in what I am trying to do, I am basically trying to dump some structs from C into a readable format for debugging purposes, this is for the "EasyCodeIt" project I am working on, see my signature.

[maniootek]

#include "JSON.au3"
Json_Test()


Func Json_Test()
    Local $sJSON = '[{"name":"Jon", "surname":"Snow"}{"name":"Daenerys","surname":"Targaryen"}]'
    Json_Dump($sJSON)
    Local $oJSON = Json_Decode($sJSON)
    Json_ObjDelete($oJSON, "[0]")
    Local $sJSON2 = Json_Encode($oJSON)
    Json_Dump($sJSON2)
EndFunc

Can someone help me to delete entry

{"name":"Jon", "surname":"Snow"}

?

[Jos]

I think the jason string is invalid and missing a comma :

Local $sJSON = '[{"name":"Jon", "surname":"Snow"},{"name":"Daenerys","surname":"Targaryen"}]'

.. but also believe the UDF has trouble lexing it as it returns an Array in stead of an Object.

[TheDcoder]

4 minutes ago, Jos said:

.. but also believe the UDF has trouble lexing it as it returns an Array in stead of an Object.

The JSON data in your post is indeed an array with two objects inside it.

[Jos]

mmm that indeed makes sense so one needs to delete the array entry 0 in this case  or just use array entry 1 works too in this example: 

#include "JSON.au3"
Json_Test()

Func Json_Test()
    Local $sJSON = '[{"name":"Jon", "surname":"Snow"},{"name":"Daenerys","surname":"Targaryen"},]'
    Json_Dump($sJSON)
    Local $oJSON = Json_Decode($sJSON)
    ConsoleWrite(Json_encode($oJSON[1]) & @crlf)
EndFunc

 

[TheXman]

If you want to programmatically search the array for the object(s) of interest, instead of hard-coding it, you could do something like:

#include <Constants.au3>
#include <Array.au3>
#Include <json.au3>

Const $JSON_DATA = '[{"name":"Jon","surname":"Snow"},{"name":"Daenerys","surname":"Targaryen"}]'

delete_json_array_items_example()

Func delete_json_array_items_example()
    Local $oJson
    Local $aoJson[0]

    ;Decode JSON
    $aoJson = Json_Decode($JSON_DATA)
    If @error Then Exit MsgBox($MB_ICONERROR + $MB_TOPMOST, "ERROR", "Unable to parse JSON - @error = " & @error)

    ConsoleWrite("Before:" & @CRLF & Json_Encode($aoJson, $JSON_PRETTY_PRINT) & @CRLF)

    ;For each JSON array item
    For $i = UBound($aoJson) - 1 To 0 Step -1
        ;Get object. If it's an object of interest, then remove it from the array
        $oJson = json_get($aoJson, "[" & $i & "]")
        If Json_Get($oJson, ".name") = "Jon" And Json_Get($oJson, ".surname") = "Snow" Then _ArrayDelete($aoJson, $i)
    Next

    ConsoleWrite(@CRLF & "After:" & @CRLF & Json_Encode($aoJson, $JSON_PRETTY_PRINT) & @CRLF)
EndFunc

Console:

Before:
[
    {
        "name": "Jon",
        "surname": "Snow"
    },
    {
        "name": "Daenerys",
        "surname": "Targaryen"
    }
]

After:
[
    {
        "name": "Daenerys",
        "surname": "Targaryen"
    }
]

 

Edited March 30, 2021 by TheXman

[Jos]

.. I do however think there should be some sort of support for deleting array entries in the UDF and we probably could even include it in the objdelete() UDF by testing whether the supplied variable is an Obj or Array....agree? ... so basically combining the existing with your logic.

Jos

Edited March 30, 2021 by Jos

[TheXman]

If that question was directed towards me:

Although I would rarely disagree that additional, relevant, functionality is a good thing, adding functionality to a UDF library when that functionality already exists seems a bit unnecessary and duplicative.  JSON arrays are still just arrays.  There's already a very effective UDF that deletes array entries, _ArrayDelete().  What would be the benefit of the json.au3 file having its own  function that basically does the same thing?  Furthermore, how would you implement it in this case where it may have multiple criteria for the selection of an item that needs to be deleted? I mean I know it can be done, but is it really worth the effort when there is already a whole UDF library dedicated to array maintenance and manipulation?

This was just my off-the-cuff opinion.  If I have misunderstood the direction or tenor of your question, please forgive me. 

Edited March 30, 2021 by TheXman

[Jos]

It was just a generic question to all but do appreciate your thoughts.  
Looking at the functions we do already have these commands for an Arrray:

• Json_get()
• Json_put()

So thought it would be nice to add Json_del() to that list as well. 
Another thought was to add a check at the beginning of eg the Json_*get() functions to check for IsObj() and isArray() and use the "other"function when it is the wrong type of input..... just to make things much easier to use it.   ( just thinking out loud here  ) 

Jos

Edited March 30, 2021 by Jos

[TheXman]

Now you are starting to get into the differences between JSON parsers, like jsmn which this UDF is based on, and JSON processors like jq.  Parsers are great for getting and, sometimes, putting specific pieces of data.  Although JSON processors have the ability to parse data too, they excel when it comes to manipulating JSON, such as in the case of deleting, adding, inserting, splicing, splitting, reformatting, and other functions changing JSON data and its structures as well as gathering information from JSON datasets by summing, averaging, and gathering other types of information from complete JSON datasets.  For most people, parsing is all they need (or think they need).  But for those that really work with JSON datasets, parsers are not powerful enough for most tasks, and that is where processors shine.

I think it would be great to add processor-type functionality to the json.au3 library.  But jsmn was not really designed for processing JSON.  So trying to take the json.au3 library from a great JSON parser to a great JSON processor would be a interesting and fun project.  But it is a whole different ballgame.  😉

Edited March 31, 2021 by TheXman

[TheXman]

@Jos

The topic below, posted by @zeehteam, identified a slight flaw in the json_get and json_put functions as it relates to handling key names that are provided using a string.  The regular expressions used to identify dot and bracket notations are a little too narrow.  Basically, the current regular expressions do not allow valid key names that have embedded right brakets ("[") in them.  Although such key names are odd and rare, they are valid.

 

The attached json udf has slightly modified versions of json_get() and json_put() for your review.  The only modification to the 2 functions is in how they identify and parse the notations in order to be more fully compliant with the standard.  Basically, it assumes that there are 4 types of notations:

1. Dot-notation using a literal value (ex.  .name)
2. Dot-notation using a string value (ex.  ."name")
3. Bracket-notation using a literal value (ex.  [name])
4. Bracket-notation using a string value (ex.  ["name"])

Indexes that are passed are correctly handled as bracket-notation using a literal (ex.  [0]).

To make it easier to maintain in the future, the only modifications to the 2 functions are at the very beginning where the notations are identified.  The rest of the logic in json_get and json_put remained the same.

 

The following example and output, using the modified functions, shows that the functions are correctly handling existing functionality and the ability to handle the new functionality.

expandcollapse popup

#AutoIt3Wrapper_AU3Check_Parameters=-w 3 -w 4 -w 5 -w 6 -d

#include <Constants.au3>
#include <MyIncludes\json\json (modified).au3> ;<== change to your location

Const $JSON = _
    '{ ' & _
    '   "project": "Listing droids", ' & _
    '   "version": "1.0", ' & _
    '   "hasErrors": false, ' & _
    '   "author": { ' & _
    '      "name": "Luke", ' & _
    '      "mail": "luke@2080.org" ' & _
    '   }, ' & _
    '   "[[[/script/droids]]]": [ ' & _
    '      { ' & _
    '         "name": "R2D2", ' & _
    '         "type": "Astromecano", ' & _
    '         "size": "0,96m" ' & _
    '      }, ' & _
    '      { ' & _
    '         "name": "BB8", ' & _
    '         "type": "Astromecano", ' & _
    '         "size": "0,67m" ' & _
    '      } ' & _
    '   ] ' & _
    '} '

json_example()

Func json_example()
    Local $oJSON
    Local $Key, $Value

    $oJSON = Json_Decode($JSON)
    If @error Then Exit MsgBox($MB_ICONERROR + $MB_TOPMOST, "ERROR", "Decode error - @error = " & @error)

    ;Get values
    $Key   = '."[[[/script/droids]]]"[0].name'
    $Value = Json_Get($oJSON, $Key)
    ConsoleWrite($Key & "   = " & $Value & @CRLF)

    $Key   = '["[[[/script/droids]]]"][0][name]'
    $Value = Json_Get($oJSON, $Key)
    ConsoleWrite($Key & " = " & $Value & @CRLF)

    $Key   = '.author."name"'
    $Value = Json_Get($oJSON, $Key)
    ConsoleWrite($Key & "   = " & $Value & @CRLF)

    $Key   = '[author]["name"]'
    $Value = Json_Get($oJSON, $Key)
    ConsoleWrite($Key & " = " & $Value & @CRLF)

    ;Put values
    $oJSON = ""
    json_put($oJSON, '.description', "test json dataset")
    json_put($oJSON, '.author', "TheXman")

    json_put($oJSON, '."[[test]]"[0].name', "test name 1")
    json_put($oJSON, '."[[test]]"[0].number', 1)
    json_put($oJSON, '."[[test]]"[0].boolean', True)
    json_put($oJSON, '."[[test]]"[0].null_value', Null)

    json_put($oJSON, '["[[test]]"][1][name]', "test name 2")
    json_put($oJSON, '["[[test]]"][1][number]', 2.5)
    json_put($oJSON, '["[[test]]"][1][boolean]', False)
    json_put($oJSON, '["[[test]]"][1][null_value]', Null)

    ConsoleWrite(@CRLF)
    ConsoleWrite(Json_Encode($oJSON, $JSON_PRETTY_PRINT) & @CRLF)
EndFunc

Output

."[[[/script/droids]]]"[0].name   = R2D2
["[[[/script/droids]]]"][0][name] = R2D2
.author."name"   = Luke
[author]["name"] = Luke

{
    "description": "test json dataset",
    "author": "TheXman",
    "[[test]]": [
        {
            "name": "test name 1",
            "number": 1,
            "boolean": true,
            "null_value": null
        },
        {
            "name": "test name 2",
            "number": 2.5,
            "boolean": false,
            "null_value": null
        }
    ]
}

 

Json (modified).au3

Edited November 18, 2021 by TheXman
Fixed typos

[Jos]

2 hours ago, TheXman said:

The attached json udf has slightly modified versions of json_get() and json_put() for your review.

I assume you tested it and all is working so just updated the first post in the thread with your update.   

[TheXman]

Yes, I tested it using the test script that comes bundled with the UDF's zip file as well as several of my own tests.

Thanks Jos! 

Edited November 18, 2021 by TheXman