Monday-paginated-board-PBI-query

.Monday-paginated-board-PBI-query

The following files are the three functions and a query that can be used to retrieve data from a Monday.com board 
using the new 2023-10 API and Cursor-based pagination.

EmptyTable.txt

let
    Source = () =>
let
  Source = #table(
    {"Column1.cursor","Column1.items.id","Column1.items.name", "Reported By", "Created Date", "AssignedTo", "State", "Priority", "Issue Category", "Due Date", "Description", "Resolution", "Resolved Date", "Project Name"},
    {
      {Text.From(null),"","", "", "", "", "", "", "", "", "", "", "", ""}
    }
  )
in
  Source
in
    Source

GetBoard_1stPage.txt

let
    Source = (BoardID as number) =>
let
    MondayQuery1 = "{""query"": ""{boards(ids: ",
    MondayQuery2 = Number.ToText(BoardID),
    MondayQuery3 = ") {items_page(limit: 100) {cursor items {id name column_values {text column {title}}}}}}""}",
    MondayFullQuery = MondayQuery1 & MondayQuery2 & MondayQuery3,
    Source = Web.Contents("https://api.monday.com/", 
        [
            RelativePath="v2",
            Headers=
            [
                #"Method"="POST", 
                #"Content-Type"="application/json", 
                #"Authorization"="eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjI1NTUwMDU0NywiYWFpIjoxMSwidWlkIjo0MzAyNDk2MiwiaWFkIjoiMjAyMy0wNS0wOVQyMzowMzowMC42NTdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTI0Mjc4NzUsInJnbiI6InVzZTEifQ.4DN9PXL0bhfbGNQ82I-Mcar-huskws2kQtuKqqpR4z0", 
                #"API-Version"="2023-10"
            ],
            Content=Text.ToBinary(MondayFullQuery)
        ]
        ),
    #"JSON" = Json.Document(Source, 65001),
    data = JSON[data],
    boards = data[boards],
    #"Converted to Table" = Table.FromList(boards, Splitter.SplitByNothing(), null, null, ExtraValues.Error),
    #"Expanded Column1" = Table.ExpandRecordColumn(#"Converted to Table", "Column1", {"items_page"}, {"Column1.items_page"}),
    #"Renamed Columns" = Table.RenameColumns(#"Expanded Column1",{{"Column1.items_page", "Column1"}}),
    #"Expanded Column2" = Table.ExpandRecordColumn(#"Renamed Columns", "Column1", {"cursor", "items"}, {"Column1.cursor", "Column1.items"}),
    #"Expanded Column1.items" = Table.ExpandListColumn(#"Expanded Column2", "Column1.items"),
    #"Expanded Column1.items1" = Table.ExpandRecordColumn(#"Expanded Column1.items", "Column1.items", {"id", "name", "column_values"}, {"Column1.items.id", "Column1.items.name", "Column1.items.column_values"}),
    #"Expanded Column1.items.column_values" = Table.ExpandListColumn(#"Expanded Column1.items1", "Column1.items.column_values"),
    #"Expanded Column1.items.column_values1" = Table.ExpandRecordColumn(#"Expanded Column1.items.column_values", "Column1.items.column_values", {"text", "column"}, {"Column1.items.column_values.text", "Column1.items.column_values.column"}),
    #"Expanded Column1.items.column_values.column" = Table.ExpandRecordColumn(#"Expanded Column1.items.column_values1", "Column1.items.column_values.column", {"title"}, {"Column1.items.column_values.column.title"}),
    #"Changed Type" = Table.TransformColumnTypes(#"Expanded Column1.items.column_values.column",{{"Column1.items.column_values.column.title", type text}}),
    #"Pivoted Column" = Table.Pivot(#"Changed Type", List.Distinct(#"Changed Type"[Column1.items.column_values.column.title]), "Column1.items.column_values.column.title", "Column1.items.column_values.text")
in
    #"Pivoted Column"
in
    Source

GetBoard_RemainingPages.txt

let
    Source = (PageCursor as text) =>
let
    Monday2ndQuery1 = "{""query"": ""{next_items_page(cursor: \""",
    Monday2ndQuery2 = PageCursor,
    Monday2ndQuery3 = "\"" limit: 100) {cursor items {id name column_values {column {title} text}}}}""}",
    MondayFull2ndQuery = Monday2ndQuery1 & Monday2ndQuery2 & Monday2ndQuery3,
    Source2 = Web.Contents("https://api.monday.com/", 
        [
            RelativePath="v2",
            Headers=
            [
                #"Method"="POST", 
                #"Content-Type"="application/json", 
                #"Authorization"="eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjI1NTUwMDU0NywiYWFpIjoxMSwidWlkIjo0MzAyNDk2MiwiaWFkIjoiMjAyMy0wNS0wOVQyMzowMzowMC42NTdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTI0Mjc4NzUsInJnbiI6InVzZTEifQ.4DN9PXL0bhfbGNQ82I-Mcar-huskws2kQtuKqqpR4z0", 
                #"API-Version"="2023-10"
            ],
            Content=Text.ToBinary(MondayFull2ndQuery)
        ]
        ),
    #"2ndJSON" = Json.Document(Source2, 65001),
    data1 = #"2ndJSON"[data],
    #"Converted to Table1" = Record.ToTable(data1),
    #"Renamed Columns1" = Table.RenameColumns(#"Converted to Table1",{{"Value", "Column1"}}),
    #"Removed Columns" = Table.RemoveColumns(#"Renamed Columns1",{"Name"}),
    #"Expanded Column2" = Table.ExpandRecordColumn(#"Removed Columns", "Column1", {"cursor", "items"}, {"Column1.cursor", "Column1.items"}),
    #"Expanded Column1.items" = Table.ExpandListColumn(#"Expanded Column2", "Column1.items"),
    #"Expanded Column1.items1" = Table.ExpandRecordColumn(#"Expanded Column1.items", "Column1.items", {"id", "name", "column_values"}, {"Column1.items.id", "Column1.items.name", "Column1.items.column_values"}),
    #"Expanded Column1.items.column_values" = Table.ExpandListColumn(#"Expanded Column1.items1", "Column1.items.column_values"),
    #"Expanded Column1.items.column_values1" = Table.ExpandRecordColumn(#"Expanded Column1.items.column_values", "Column1.items.column_values", {"column", "text"}, {"Column1.items.column_values.column", "Column1.items.column_values.text"}),
    #"Expanded Column1.items.column_values.column" = Table.ExpandRecordColumn(#"Expanded Column1.items.column_values1", "Column1.items.column_values.column", {"title"}, {"Column1.items.column_values.column.title"}),
    #"Changed Type1" = Table.TransformColumnTypes(#"Expanded Column1.items.column_values.column",{{"Column1.items.column_values.column.title", type text}}),
    #"Pivoted Column1" = Table.Pivot(#"Changed Type1", List.Distinct(#"Changed Type1"[Column1.items.column_values.column.title]), "Column1.items.column_values.column.title", "Column1.items.column_values.text")
in
    #"Pivoted Column1"
in
    Source

Monday Board query.txt

let 
    	/* The Monday.com API dated 2023-10 changed the way rows of data are returned from boards.  You now have to call for 1 page of data
    	at a time and also request the cursor which points to the next page of data.  If the cursor is null there are no more pages of data.
    	The maximum number of rows per page is configurable with the maximum allow of 100 rows of data.  The following query attempts to 
    	request five pages of 100 rows of data.  If you expect more than 500 rows of data then additional page requests need to be added below. */
    	
    	
    	/* Define the Monday.com board ID number then call the GetBoard_1stPage function to return the first 100 rows of data */
    	/* Update the Column1.cursor column if the value is null and replace with NoMorePages  */
    	/* Assign the value found in the Column1.cursor column to the 2nd_page_cursor variable */
    
    BoardID = 1234567890,
    #"1st_Page_Table" = GetBoard_1stPage(BoardID),
    #"1st_Page_Table_Updated" = Table.ReplaceValue(#"1st_Page_Table",null,"NoMorePages",Replacer.ReplaceValue,{"Column1.cursor"}),
    #"2nd_page_cursor" = #"1st_Page_Table_Updated"{0}[Column1.cursor],

		/* If the 2nd_page_cursor value is not equal to NoMorePages then call the GetBoard_RemainingPages function to get the next page of 100 rows of data */
		/* If the cursor value is NoMorePages then call the EmptyIssuesTable to create an empty table with the correct column names */
		/* Update the Column1.cursor column if the value is null and replace with NoMorePages  */
		/* Assign the value found in the Column1.cursor column to the 3rd_page_cursor variable */


    #"2nd_Page_Table" = if #"2nd_page_cursor" <> "NoMorePages" then GetBoard_RemainingPages(#"2nd_page_cursor") else EmptyTable(),
    #"2nd_Page_Table_Updated" = Table.ReplaceValue(#"2nd_Page_Table",null,"NoMorePages",Replacer.ReplaceValue,{"Column1.cursor"}),
    #"3rd_page_cursor" = #"2nd_Page_Table_Updated"{0}[Column1.cursor],

		/* If the 3rd_page_cursor value is not equal to NoMorePages then call the GetBoard_RemainingPages function to get the next page of 100 rows of data */
		/* If the cursor value is NoMorePages then call the EmptyIssuesTable to create an empty table with the correct column names */
		/* Update the Column1.cursor column if the value is null and replace with NoMorePages  */
		/* Assign the value found in the Column1.cursor column to the 4th_page_cursor variable */


    #"3rd_Page_Table" = if #"3rd_page_cursor" <> "NoMorePages" then GetBoard_RemainingPages(#"3rd_page_cursor") else EmptyTable(),
    #"3rd_Page_Table_Updated" = Table.ReplaceValue(#"3rd_Page_Table",null,"NoMorePages",Replacer.ReplaceValue,{"Column1.cursor"}),
    #"4th_page_cursor" = #"3rd_Page_Table_Updated"{0}[Column1.cursor],

		/* If the 4th_page_cursor value is not equal to NoMorePages then call the GetBoard_RemainingPages function to get the next page of 100 rows of data */
		/* If the cursor value is NoMorePages then call the EmptyIssuesTable to create an empty table with the correct column names */
		/* Update the Column1.cursor column if the value is null and replace with NoMorePages  */
		/* Assign the value found in the Column1.cursor column to the 5th_page_cursor variable */


    #"4th_Page_Table" = if #"4th_page_cursor" <> "NoMorePages" then GetBoard_RemainingPages(#"4th_page_cursor") else EmptyTable(),
    #"4th_Page_Table_Updated" = Table.ReplaceValue(#"4th_Page_Table",null,"NoMorePages",Replacer.ReplaceValue,{"Column1.cursor"}),
    #"5th_page_cursor" = #"4th_Page_Table_Updated"{0}[Column1.cursor],

		/* If the 5th_page_cursor value is not equal to NoMorePages then call the GetBoard_RemainingPages function to get the next page of 100 rows of data */
		/* If the cursor value is NoMorePages then call the EmptyIssuesTable to create an empty table with the correct column names */
		/* Update the Column1.cursor column if the value is null and replace with NoMorePages  */
		/* Assign the value found in the Column1.cursor column to the 6th_page_cursor variable */


    #"5th_Page_Table" = if #"5th_page_cursor" <> "NoMorePages" then GetBoard_RemainingPages(#"5th_page_cursor") else EmptyTable(),
    #"5th_Page_Table_Updated" = Table.ReplaceValue(#"5th_Page_Table",null,"NoMorePages",Replacer.ReplaceValue,{"Column1.cursor"}),
    #"6th_page_cursor" = #"5th_Page_Table_Updated"{0}[Column1.cursor],

		/* If you need to load more than 500 rows of data then duplicate the three lines above and modify the variable names.  You will also need to add the new variable in the Table.Combine function below */
		/*  E.G.
		
		    #"6th_Page_Table" = if #"6th_page_cursor" <> "NoMorePages" then GetBoard_RemainingPages(#"6th_page_cursor") else EmptyTable(),
    		#"6th_Page_Table_Updated" = Table.ReplaceValue(#"6th_Page_Table",null,"NoMorePages",Replacer.ReplaceValue,{"Column1.cursor"}),
    		#"7th_page_cursor" = #"6th_Page_Table_Updated"{0}[Column1.cursor],
		*/


		/* Combine the five tables contained in #"1st_Page_Table_Updated", #"2nd_Page_Table_Updated", #"3rd_Page_Table_Updated", #"4th_Page_Table_Updated", #"5th_Page_Table_Updated" */
		/* Keep all columns except the Column1.cursor and Column1.items.id columns.  These two columns are not needed.  */
		/* Rename the Column1.items.name to Topic */
		/* Remove the empty rows that were due to any empty tables. */


    Combined_Table1 = Table.Combine({#"1st_Page_Table_Updated", #"2nd_Page_Table_Updated", #"3rd_Page_Table_Updated", #"4th_Page_Table_Updated", #"5th_Page_Table_Updated"}),    // add additional table variables if more than 500 rows of data
    #"Removed Other Columns" = Table.SelectColumns(Combined_Table1,{"Column1.items.name", "Reported By", "Created Date", "AssignedTo", "State", "Priority", "Issue Category", "Due Date", "Description", "Resolution", "Resolved Date", "Project Name"}),
    #"Renamed Columns" = Table.RenameColumns(#"Removed Other Columns",{{"Column1.items.name", "Item"}}),
    #"Filtered Rows" = Table.SelectRows(#"Renamed Columns", each [Item] <> null and [Item] <> "")
in
    #"Filtered Rows"

I ran into an error when I was trying this method out. The error I see in the Monday Board Query table is: Formula.Firewall: Query 'Table' (step '2nd_Page_Table_Updated') references other queries or steps, so it may not directly access a data source. Please rebuild this data combination. Any suggestions as to how to resolve this error?

Resolved it by going into the File -> Options and Settings -> Options -> Current File -> Privacy -> Selected Ignore the Privacy Levels and potentially improve performance

Exactly. You have to ignore the privacy levels in your PBIX file.

Thanks for this @hmbouk.
I was able to get the groups in the same query without the need for additional query.
https://github.com/sachecka/Monday-PBI-Connection

Perfect. Glad you were able to figure this out @sachecka. I have been working on a new version of queries that can loop and process large boards. I chose to bring in the main board data in one query, groups in another query and subitems in a third query. Then I join them (if needed) in Power BI. Hopefully I will finish debugging the queries soon. Howard Bouk Technology and Project Management Consultant T. 416-806-7353 E: ***@***.*** W: www.hmbconsulting.ca<http://www.hmbconsulting.ca/> Port Hope, ON

…

________________________________ From: sachecka ***@***.***> Sent: Sunday, January 21, 2024 7:55:27 PM To: sachecka ***@***.***> Cc: Mention ***@***.***> Subject: Re: hmbouk/.Monday-paginated-board-PBI-query @sachecka commented on this gist.

________________________________ Thanks for this @hmbouk<https://github.com/hmbouk>. I was able to get the groups in the same query without the need for additional query. https://github.com/sachecka/Monday-PBI-Connection — Reply to this email directly, view it on GitHub<https://gist.github.com/hmbouk/b5712f8d4639eb85328bf4828b402284#gistcomment-4842737> or unsubscribe<https://github.com/notifications/unsubscribe-auth/ADGMAVQEGHQYZXQTUA47HDTYPW2IBBFKMF2HI4TJMJ2XIZLTSKBKK5TBNR2WLJDUOJ2WLJDOMFWWLO3UNBZGKYLEL5YGC4TUNFRWS4DBNZ2F6YLDORUXM2LUPGBKK5TBNR2WLJDHNFZXJJDOMFWWLK3UNBZGKYLEL52HS4DFVRZXKYTKMVRXIX3UPFYGLK2HNFZXIQ3PNVWWK3TUUZ2G64DJMNZZDAVEOR4XAZNEM5UXG5FFOZQWY5LFVEYTENRRGU3DKMRVU52HE2LHM5SXFJTDOJSWC5DF>. You are receiving this email because you were mentioned. Triage notifications on the go with GitHub Mobile for iOS<https://apps.apple.com/app/apple-store/id1477376905?ct=notification-email&mt=8&pt=524675> or Android<https://play.google.com/store/apps/details?id=com.github.android&referrer=utm_campaign%3Dnotification-email%26utm_medium%3Demail%26utm_source%3Dgithub>.

Thanks for this @hmbouk
I was able to adapt your queries and get the dynamic paging implemented for my use case.
I have published these on https://github.com/ariten/Monday-to-PowerBi
It can be quite slow if you have a lot of records but the dynamic paging does work for me.