Skip to content

Instantly share code, notes, and snippets.

@7effrey89

7effrey89/Variance Analysis.ipynb

Last active October 21, 2025 12:57
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save 7effrey89/02dce3575b890e8059a9fdd509fd98a2 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save 7effrey89/02dce3575b890e8059a9fdd509fd98a2 to your computer and use it in GitHub Desktop.
Download ZIP
Fabric - Loop through each row in table and search column values against Azure AI Search, then use AI Functions to summarize output in new column
Display the source blob
Display the rendered blob
Raw
Variance Analysis.ipynb
{
"cells": [
{
"cell_type": "markdown",
"id": "80a0abf6-1d24-4313-8642-0184c2e15806",
"metadata": {
"microsoft": {
"language": "python",
"language_group": "synapse_pyspark"
},
"nteract": {
"transient": {
"deleting": false
}
}
},
"source": [
"# Demo table"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5a2951c6-4a8f-45cb-a82d-88a623a8cfce",
"metadata": {
"collapsed": false,
"microsoft": {
"language": "python",
"language_group": "synapse_pyspark"
}
},
"outputs": [],
"source": [
"df = spark.createDataFrame([\n",
" (\"København\", \"2022\"),\n",
" (\"Århus\", \"2021\"),\n",
" (\"Aalborg\", \"2021\"),\n",
" (\"Odense\", \"2020\")\n",
"], [\"municipality\", \"year\"])\n",
"\n",
"display(df)"
]
},
{
"cell_type": "markdown",
"id": "d64c0f9b-874d-468e-aa2e-338c4e085823",
"metadata": {
"microsoft": {
"language": "python",
"language_group": "synapse_pyspark"
},
"nteract": {
"transient": {
"deleting": false
}
}
},
"source": [
"## Search for relevant chunks"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "970fa5e7-6ee6-464d-ac57-c4bb4d760ff5",
"metadata": {
"collapsed": false,
"jupyter": {
"source_hidden": false
},
"microsoft": {
"language": "python",
"language_group": "synapse_pyspark"
}
},
"outputs": [],
"source": [
"#### ATTENTION: AI-generated code can include errors or operations you didn't intend. Review the code in this cell carefully before running it.\n",
"\n",
"import pandas as pd\n",
"import requests\n",
"import time\n",
"\n",
"# Convert Spark DataFrame to pandas DataFrame\n",
"pdf = df.toPandas()\n",
"\n",
"service_name = \"jlas#############aisearch\"\n",
"index_name = \"####################\"\n",
"ai_search_key = \"##########################\"\n",
"url = f\"https://{service_name}.search.windows.net/indexes/{index_name}/docs/search?api-version=2025-08-01-preview\"\n",
"\n",
"def fetch_result(municipality, year):\n",
" search_term = municipality\n",
" payload = {\n",
" \"search\": search_term,\n",
" \"count\": True,\n",
" \"vectorQueries\": [\n",
" {\n",
" \"kind\": \"text\",\n",
" \"text\": search_term,\n",
" \"fields\": \"text_vector\"\n",
" }\n",
" ],\n",
" \"queryType\": \"semantic\",\n",
" \"semanticConfiguration\": \"rag-kommune-semantic-configuration\",\n",
" \"captions\": \"extractive\",\n",
" \"answers\": \"extractive|count-3\",\n",
" \"queryLanguage\": \"en-us\"\n",
" }\n",
" try:\n",
" response = requests.post(\n",
" url,\n",
" json=payload,\n",
" headers={\"api-key\": ai_search_key}\n",
" )\n",
" resp_json = response.json()\n",
" time.sleep(1) # Avoid rate-limiting\n",
" return resp_json\n",
" except Exception as e:\n",
" time.sleep(1)\n",
" return str(e)\n",
"\n",
"def extract_ai_text_score(d, max_length=256000):\n",
" \"\"\"\n",
" Extract all 'text' fields and 'score' from '@search.answers' in the search result dict.\n",
" Concatenate with a separator. Truncate the result if longer than max_length.\n",
" \"\"\"\n",
" if isinstance(d, dict) and \"@search.answers\" in d:\n",
" answers = d[\"@search.answers\"]\n",
" if isinstance(answers, list) and answers:\n",
" formatted_answers = []\n",
" for ans in answers:\n",
" if isinstance(ans, dict):\n",
" score = ans.get(\"score\", \"n/a\")\n",
" text = ans.get(\"text\", \"\")\n",
" formatted_answers.append(f\"Score: {score}\\nText: {text}\")\n",
" result = \"\\n\\n\".join(formatted_answers)\n",
" return result[:max_length]\n",
" return \"\"\n",
"\n",
"# Run API query per row and save as 'search_result'\n",
"pdf['search_result'] = pdf.apply(lambda row: fetch_result(row['municipality'], row['year']), axis=1)\n",
"\n",
"# Extract all answer texts with scores\n",
"pdf['ai_search_results'] = pdf['search_result'].apply(extract_ai_text_score)\n",
"\n",
"# print(pdf[['municipality', 'ai_search_results']])\n",
"\n",
"# If you want a Spark DataFrame:\n",
"df_new = spark.createDataFrame(pdf[[\"municipality\", \"year\", \"ai_search_results\"]])\n",
"display(df_new)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4dd61151-9471-44cb-b8de-ad1ab6e87f8b",
"metadata": {
"microsoft": {
"language": "python",
"language_group": "synapse_pyspark"
}
},
"outputs": [],
"source": [
"import synapse.ml.spark.aifunc as aifunc"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5209fab6-fe53-461e-bae0-d9a80dea1475",
"metadata": {
"collapsed": false,
"microsoft": {
"language": "python",
"language_group": "synapse_pyspark"
}
},
"outputs": [],
"source": [
"# This code uses AI. Always review output for mistakes. \n",
"# Read terms: https://azure.microsoft.com/support/legal/preview-supplemental-terms/.\n",
"\n",
"responses = df_new.ai.generate_response(prompt=\"you are a helpful ai agent. Use the following context to explain the highlights for the year: {ai_search_results}. provide a short summary in 2 sentences\", is_prompt_template=True, output_col=\"ai_response\")\n",
"display(responses)\n"
]
}
],
"metadata": {
"dependencies": {},
"kernel_info": {
"name": "synapse_pyspark"
},
"kernelspec": {
"display_name": "synapse_pyspark",
"name": "synapse_pyspark"
},
"language_info": {
"name": "python"
},
"microsoft": {
"language": "python",
"language_group": "synapse_pyspark",
"ms_spell_check": {
"ms_spell_check_language": "en"
}
},
"nteract": {
"version": "nteract-front-end@1.0.0"
},
"spark_compute": {
"compute_id": "/trident/default",
"session_options": {
"conf": {
"spark.synapse.nbs.session.timeout": "1200000"
}
}
},
"synapse_widget": {
"state": {
"0c3a1dd3-974a-4192-97ed-55a05b535b58": {
"persist_state": {
"view": {
"chartOptions": {
"aggregationType": "sum",
"binsNumber": 10,
"categoryFieldKeys": [],
"chartType": "bar",
"isStacked": false,
"seriesFieldKeys": [],
"wordFrequency": "-1"
},
"tableOptions": {},
"type": "details",
"viewOptionsGroup": [
{
"tabItems": [
{
"key": "0",
"name": "Table",
"options": {},
"type": "table"
}
]
}
]
}
},
"sync_state": {
"isSummary": false,
"language": "scala",
"table": {
"rows": [
{
"0": "København",
"1": "2022"
},
{
"0": "Århus",
"1": "2021"
},
{
"0": "Aalborg",
"1": "2021"
},
{
"0": "Odense",
"1": "2020"
}
],
"schema": [
{
"key": "0",
"name": "municipality",
"type": "string"
},
{
"key": "1",
"name": "year",
"type": "string"
}
],
"truncated": false
},
"wranglerEntryContext": {
"candidateVariableNames": [
"df"
],
"dataframeType": "pyspark"
}
},
"type": "Synapse.DataFrame"
},
"620c2e50-fe82-406c-baa9-bd362eb9cc3e": {
"persist_state": {
"view": {
"chartOptions": {
"aggregationType": "sum",
"binsNumber": 10,
"categoryFieldKeys": [],
"chartType": "bar",
"isStacked": false,
"seriesFieldKeys": [],
"wordFrequency": "-1"
},
"tableOptions": {},
"type": "details",
"viewOptionsGroup": [
{
"tabItems": [
{
"key": "0",
"name": "Table",
"options": {},
"type": "table"
}
]
}
]
}
},
"sync_state": {
"isSummary": false,
"language": "scala",
"table": {
"rows": [
{
"0": "København",
"1": "2022",
"2": "Score: 0.9480000138282776\nText: - **Proximity to Copenhagen**: Quick access to the metropolis while enjoying the tranquility of suburban life. Overall, Lyngby-Taarbæk presents a compelling case for relocation, offering both urban convenience and suburban charm, along with exceptional educational and recreational facilities. The balance of a high standard of living, safety, and...\n\nScore: 0.9129999876022339\nText: - **Historical Significance**: Historical streets and buildings echo its rich Viking and medieval history. ### Other Compelling Reasons to Move - **Safety and Security**: Known for low crime rates and high levels of public safety. - **Community Engagement**: Strong community programs and volunteer opportunities make it easy to connect. ...ciety**:\n\nScore: 0.8299999833106995\nText: public healthcare system. - **Libraries**: The Frederiksberg Public Library system offers access to various resources, events, and reading programs throughout the municipality. Proximity to Copenhagen also offers commuting options to a wider job market. #### Key Highlights and Unique Reasons to Move There - **Green Spaces**: Frederiksberg is kno..."
},
{
"0": "Århus",
"1": "2021",
"2": "Score: 0.9850000143051147\nText: ### Transportation Options - **Public Transport**: Silkeborg is well-connected by bus services to nearby cities and towns. - **Roads**: Major roads and highways, including the E45 motorway, provide easy access to Aarhus and other significant cities. - **Cycling**: A robust network of cycling paths encourages biking throughout the town and surrou...\n\nScore: 0.9829999804496765\nText: - **International Schools**: Limited international schooling options, b... - **University Access**: Proximity to Aarhus and other cities with higher education facilities. --- #### What the Municipality is Especially Known For - **Cultural Heritage**: Rich history with several historic sites, showcasing Denmark's medieval architecture and heritage.\n\nScore: 0.9829999804496765\nText: ### Overview of Norddjurs Municipality Norddjurs is a picturesque municipality located in central Jutland, Denmark, characterized by its charming coastal villages, rich cul... - **Cost of Living**: Generally lower than major Danish cities like Copenhagen and Aarhus, making it an attractive option for families and individuals seeking affordability."
},
{
"0": "Aalborg",
"1": "2021",
"2": "Score: 0.9879999756813049\nText: Danish history and culture. ## Transportation Options - **Public Transport**: Bus services connect Rebild to nearby towns and the larger city of Aalborg. - **International Options**: While there are limited international schools within Rebild, nearby Aalborg houses international educational institutions for expatriates. - **Community Festivals*...\n\nScore: 0.9860000014305115\nText: The Viborg art museum showcasing local and international artworks. ## Transportation Options - **Public Transport**: - Well-connected bus networks for easy access to nearb... - Trains available, connecting Viborg with larger cities like Aarhus and Aalborg. - **Roads**: - Major roads and highways provide convenient access to surrounding areas.\n\nScore: 0.953000009059906\nText: - **Animal and Nature Lover Activities**: Equ... ### 9. Transportation Options - **Public Transport**: Good bus connections between towns and to larger cities, including Aalborg. - **Roads**: Well-maintained road network with easy access to national routes. - **Cycling**: Extensive cycling paths and facilities promoting a bike-friendly environment."
},
{
"0": "Odense",
"1": "2020",
"2": "Score: 0.9909999966621399\nText: - **Cultural Events**: Numerous festivals, music concerts, and art exhibitions hel... ### Transportation Options - **Public Transport**: Efficient local bus service connects Middelfart to neighboring cities like Fredericia and Odense. - **Road Access**: Well-connected via the E20 motorway, making it convenient for commuting to larger urban centers.\n\nScore: 0.9909999966621399\nText: - **Higher Education**: The town is in proximity to Odense, which has several colleges and universities for higher education opportunities. #### Special... In summary, Nyborg Municipality offers a unique blend of historical charm, modern amenities, and a strong community friendly environment, making it a compelling choice for relocation to Denmark.\n\nScore: 0.9909999966621399\nText: - **Cultural Events**: Numerous festivals, music concerts, and art exhibitions hel... ### Transportation Options - **Public Transport**: Efficient local bus service connects Middelfart to neighboring cities like Fredericia and Odense. - **Road Access**: Well-connected via the E20 motorway, making it convenient for commuting to larger urban centers."
}
],
"schema": [
{
"key": "0",
"name": "municipality",
"type": "string"
},
{
"key": "1",
"name": "year",
"type": "string"
},
{
"key": "2",
"name": "ai_search_results",
"type": "string"
}
],
"truncated": false
},
"wranglerEntryContext": {
"candidateVariableNames": [
"df_new"
],
"dataframeType": "pyspark"
}
},
"type": "Synapse.DataFrame"
},
"a63790eb-b3e0-4869-b812-6d1c498731b7": {
"persist_state": {
"view": {
"chartOptions": {
"aggregationType": "sum",
"binsNumber": 10,
"categoryFieldKeys": [],
"chartType": "bar",
"isStacked": false,
"seriesFieldKeys": [],
"wordFrequency": "-1"
},
"tableOptions": {},
"type": "details",
"viewOptionsGroup": [
{
"tabItems": [
{
"key": "0",
"name": "Table",
"options": {},
"type": "table"
}
]
}
]
}
},
"sync_state": {
"isSummary": false,
"language": "scala",
"table": {
"rows": [
{
"0": "Microsoft Teams",
"1": "2017",
"2": "\n The ultimate messaging app for your organization—a workspace for real-time \n collaboration and communication, meetings, file and app sharing, and even the \n occasional emoji! All in one place, all in the open, all accessible to everyone.\n ",
"3": "The ultimate messaging app for organizations offers real-time collaboration, communication, meetings, file and app sharing, and emoji use, all in one accessible workspace."
},
{
"0": "Microsoft Fabric",
"1": "2023",
"2": "\n An enterprise-ready, end-to-end analytics platform that unifies data movement, \n data processing, ingestion, transformation, and report building into a seamless, \n user-friendly SaaS experience. Transform raw data into actionable insights.\n ",
"3": "An enterprise-ready analytics platform that integrates data movement, processing, ingestion, transformation, and report building into a user-friendly SaaS experience, turning raw data into actionable insights."
}
],
"schema": [
{
"key": "0",
"name": "product",
"type": "string"
},
{
"key": "1",
"name": "release_year",
"type": "string"
},
{
"key": "2",
"name": "description",
"type": "string"
},
{
"key": "3",
"name": "summary",
"type": "string"
},
{
"key": "4",
"name": "description_summarize_error",
"type": "StructType(StructField(response,StringType,true),StructField(status,StructType(StructField(protocolVersion,StructType(StructField(protocol,StringType,true),StructField(major,IntegerType,false),StructField(minor,IntegerType,false)),true),StructField(statusCode,IntegerType,false),StructField(reasonPhrase,StringType,true)),true))"
}
],
"truncated": false
},
"wranglerEntryContext": {
"candidateVariableNames": [
"summaries"
],
"dataframeType": "pyspark"
}
},
"type": "Synapse.DataFrame"
},
"dcf6457c-e22d-4ee5-9123-3fa881076adc": {
"persist_state": {
"view": {
"chartOptions": {
"aggregationType": "sum",
"binsNumber": 10,
"categoryFieldKeys": [],
"chartType": "bar",
"isStacked": false,
"seriesFieldKeys": [],
"wordFrequency": "-1"
},
"tableOptions": {},
"type": "details",
"viewOptionsGroup": [
{
"tabItems": [
{
"key": "0",
"name": "Table",
"options": {},
"type": "table"
}
]
}
]
}
},
"sync_state": {
"isSummary": false,
"language": "scala",
"table": {
"rows": [
{
"0": "København",
"1": "2022",
"2": "Score: 0.9480000138282776\nText: - **Proximity to Copenhagen**: Quick access to the metropolis while enjoying the tranquility of suburban life. Overall, Lyngby-Taarbæk presents a compelling case for relocation, offering both urban convenience and suburban charm, along with exceptional educational and recreational facilities. The balance of a high standard of living, safety, and...\n\nScore: 0.9129999876022339\nText: - **Historical Significance**: Historical streets and buildings echo its rich Viking and medieval history. ### Other Compelling Reasons to Move - **Safety and Security**: Known for low crime rates and high levels of public safety. - **Community Engagement**: Strong community programs and volunteer opportunities make it easy to connect. ...ciety**:\n\nScore: 0.8299999833106995\nText: public healthcare system. - **Libraries**: The Frederiksberg Public Library system offers access to various resources, events, and reading programs throughout the municipality. Proximity to Copenhagen also offers commuting options to a wider job market. #### Key Highlights and Unique Reasons to Move There - **Green Spaces**: Frederiksberg is kno...",
"3": "In 2023, Lyngby-Taarbæk emerged as an attractive relocation destination, offering a blend of urban convenience and suburban tranquility, alongside exceptional educational and recreational facilities. Meanwhile, Frederiksberg highlighted its rich historical significance, low crime rates, strong community engagement, and access to green spaces, making it a desirable place to live with a robust public healthcare system and library resources."
},
{
"0": "Århus",
"1": "2021",
"2": "Score: 0.9850000143051147\nText: ### Transportation Options - **Public Transport**: Silkeborg is well-connected by bus services to nearby cities and towns. - **Roads**: Major roads and highways, including the E45 motorway, provide easy access to Aarhus and other significant cities. - **Cycling**: A robust network of cycling paths encourages biking throughout the town and surrou...\n\nScore: 0.9829999804496765\nText: - **International Schools**: Limited international schooling options, b... - **University Access**: Proximity to Aarhus and other cities with higher education facilities. --- #### What the Municipality is Especially Known For - **Cultural Heritage**: Rich history with several historic sites, showcasing Denmark's medieval architecture and heritage.\n\nScore: 0.9829999804496765\nText: ### Overview of Norddjurs Municipality Norddjurs is a picturesque municipality located in central Jutland, Denmark, characterized by its charming coastal villages, rich cul... - **Cost of Living**: Generally lower than major Danish cities like Copenhagen and Aarhus, making it an attractive option for families and individuals seeking affordability.",
"3": "In 2023, Silkeborg and Norddjurs Municipality highlighted their excellent transportation options, including well-connected public transport, major road access, and extensive cycling paths, making them accessible and bike-friendly. Additionally, Norddjurs is known for its rich cultural heritage, picturesque coastal villages, and a lower cost of living compared to larger Danish cities, appealing to families and individuals seeking affordability."
},
{
"0": "Aalborg",
"1": "2021",
"2": "Score: 0.9879999756813049\nText: Danish history and culture. ## Transportation Options - **Public Transport**: Bus services connect Rebild to nearby towns and the larger city of Aalborg. - **International Options**: While there are limited international schools within Rebild, nearby Aalborg houses international educational institutions for expatriates. - **Community Festivals*...\n\nScore: 0.9860000014305115\nText: The Viborg art museum showcasing local and international artworks. ## Transportation Options - **Public Transport**: - Well-connected bus networks for easy access to nearb... - Trains available, connecting Viborg with larger cities like Aarhus and Aalborg. - **Roads**: - Major roads and highways provide convenient access to surrounding areas.\n\nScore: 0.953000009059906\nText: - **Animal and Nature Lover Activities**: Equ... ### 9. Transportation Options - **Public Transport**: Good bus connections between towns and to larger cities, including Aalborg. - **Roads**: Well-maintained road network with easy access to national routes. - **Cycling**: Extensive cycling paths and facilities promoting a bike-friendly environment.",
"3": "In 2023, Rebild and Viborg showcased their rich cultural heritage through community festivals and art exhibitions, with the Viborg art museum highlighting both local and international artworks. Transportation options in both areas were enhanced by well-connected public transport systems, including buses and trains, as well as extensive cycling paths, facilitating easy access to nearby towns and larger cities like Aalborg."
},
{
"0": "Odense",
"1": "2020",
"2": "Score: 0.9909999966621399\nText: - **Cultural Events**: Numerous festivals, music concerts, and art exhibitions hel... ### Transportation Options - **Public Transport**: Efficient local bus service connects Middelfart to neighboring cities like Fredericia and Odense. - **Road Access**: Well-connected via the E20 motorway, making it convenient for commuting to larger urban centers.\n\nScore: 0.9909999966621399\nText: - **Higher Education**: The town is in proximity to Odense, which has several colleges and universities for higher education opportunities. #### Special... In summary, Nyborg Municipality offers a unique blend of historical charm, modern amenities, and a strong community friendly environment, making it a compelling choice for relocation to Denmark.\n\nScore: 0.9909999966621399\nText: - **Cultural Events**: Numerous festivals, music concerts, and art exhibitions hel... ### Transportation Options - **Public Transport**: Efficient local bus service connects Middelfart to neighboring cities like Fredericia and Odense. - **Road Access**: Well-connected via the E20 motorway, making it convenient for commuting to larger urban centers.",
"3": "In 2023, Middelfart and Nyborg Municipality showcased a vibrant cultural scene with numerous festivals, music concerts, and art exhibitions, enhancing community engagement. The towns also benefited from efficient public transport and road access, making them attractive options for residents and commuters alike."
}
],
"schema": [
{
"key": "0",
"name": "municipality",
"type": "string"
},
{
"key": "1",
"name": "year",
"type": "string"
},
{
"key": "2",
"name": "ai_search_results",
"type": "string"
},
{
"key": "3",
"name": "ai_response",
"type": "string"
},
{
"key": "4",
"name": "generate_response_error",
"type": "StructType(StructField(response,StringType,true),StructField(status,StructType(StructField(protocolVersion,StructType(StructField(protocol,StringType,true),StructField(major,IntegerType,false),StructField(minor,IntegerType,false)),true),StructField(statusCode,IntegerType,false),StructField(reasonPhrase,StringType,true)),true))"
}
],
"truncated": false
},
"wranglerEntryContext": {
"candidateVariableNames": [
"responses"
],
"dataframeType": "pyspark"
}
},
"type": "Synapse.DataFrame"
}
},
"version": "0.1"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment