Fabric - AI function demo

notebook.ipynb

{"cells":[{"cell_type":"code","source":["import synapse.ml.spark.aifunc as aifunc"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":3,"statement_ids":[3],"state":"finished","livy_statement_state":"available","session_id":"b06483b8-06e7-49af-8469-cd73b94f47c0","normalized_state":"finished","queued_time":"2025-10-21T13:03:51.9259028Z","session_start_time":"2025-10-21T13:03:51.9268839Z","execution_start_time":"2025-10-21T13:06:34.8070638Z","execution_finish_time":"2025-10-21T13:06:38.7327512Z","parent_msg_id":"23f03765-30c2-483e-8aed-b478eb533731"},"text/plain":"StatementMeta(, b06483b8-06e7-49af-8469-cd73b94f47c0, 3, Finished, Available, Finished)"},"metadata":{}}],"execution_count":2,"metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"3da57762-fe0f-4ee9-a2e8-fc6bbaa2184a"},{"cell_type":"code","source":["df = spark.createDataFrame([\n","    (\"København\", \"2022\"),\n","    (\"Århus\", \"2021\"),\n","    (\"Aalborg\", \"2021\"),\n","    (\"Odense\", \"2020\")\n","], [\"municipality\", \"year\"])\n","\n","responses = df.ai.generate_response(prompt=\"you are a helpful ai agent. Use the following context to explain the highlights for the year: {year} for municipality {municipality}. provide a short summary in 2 sentences\", is_prompt_template=True, output_col=\"ai_response\")\n","display(responses)"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":16,"statement_ids":[16],"state":"finished","livy_statement_state":"available","session_id":"b06483b8-06e7-49af-8469-cd73b94f47c0","normalized_state":"finished","queued_time":"2025-10-21T13:13:03.3812246Z","session_start_time":null,"execution_start_time":"2025-10-21T13:13:03.3823584Z","execution_finish_time":"2025-10-21T13:13:21.271517Z","parent_msg_id":"876a3472-90ec-480c-b085-f8ed26f5ff85"},"text/plain":"StatementMeta(, b06483b8-06e7-49af-8469-cd73b94f47c0, 16, Finished, Available, Finished)"},"metadata":{}},{"output_type":"display_data","data":{"application/vnd.synapse.widget-view+json":{"widget_id":"0a829a64-fdd7-4b47-a638-849212ca7922","widget_type":"Synapse.DataFrame"},"text/plain":"SynapseWidget(Synapse.DataFrame, 0a829a64-fdd7-4b47-a638-849212ca7922)"},"metadata":{}}],"execution_count":15,"metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"},"collapsed":false},"id":"e3aa070a-2397-4eda-af6a-e28bba462282"},{"cell_type":"markdown","source":["other:\n","\n","Calculate similarity with ai.similarity: Compare the meaning of input text with a single common text value, or with corresponding text values in another column.\n","Categorize text with ai.classify: Classify input text values according to labels you choose.\n","Detect sentiment with ai.analyze_sentiment: Identify the emotional state expressed by input text.\n","Extract entities with ai.extract: Find and extract specific types of information from input text (for example, locations or names).\n","Fix grammar with ai.fix_grammar: Correct the spelling, grammar, and punctuation of input text.\n","Summarize text with ai.summarize: Get summaries of input text.\n","Translate text with ai.translate: Translate input text into another language.\n","Answer custom user prompts with ai.generate_response: Generate responses based on your own instructions.\n","\n","https://learn.microsoft.com/en-us/fabric/data-science/ai-functions/overview?tabs=pandas-pyspark%2Cpandas"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"49bb3e1d-9b79-45d3-af55-e52ea5648b43"}],"metadata":{"language_info":{"name":"python"},"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"display_name":"synapse_pyspark","language":null,"name":"synapse_pyspark"},"a365ComputeOptions":null,"sessionKeepAliveTimeout":0,"microsoft":{"language":"python","language_group":"synapse_pyspark","ms_spell_check":{"ms_spell_check_language":"en"}},"nteract":{"version":"nteract-front-end@1.0.0"},"spark_compute":{"compute_id":"/trident/default","session_options":{"conf":{"spark.synapse.nbs.session.timeout":"1200000"}}},"synapse_widget":{"version":"0.1","state":{"0a829a64-fdd7-4b47-a638-849212ca7922":{"type":"Synapse.DataFrame","sync_state":{"table":{"rows":[{"0":"København","1":"2022","2":"In 2022, the municipality of København focused on sustainable urban development and enhancing public transportation to improve residents' quality of life. Key initiatives included expanding green spaces and investing in eco-friendly infrastructure to support the city's climate goals.","key":0,"index":0},{"0":"Århus","1":"2021","2":"In 2021, the municipality of Århus focused on sustainable development and enhancing public services, including investments in green infrastructure and digitalization. The year also saw increased community engagement initiatives aimed at improving quality of life for residents.","key":1,"index":1},{"0":"Aalborg","1":"2021","2":"In 2021, Aalborg municipality focused on sustainable development and enhancing community well-being, implementing green initiatives and improving public services. The year also saw significant investments in infrastructure and cultural projects, strengthening Aalborg's position as a vibrant and forward-looking city.","key":2,"index":2},{"0":"Odense","1":"2020","2":"In 2020, the municipality of Odense focused on enhancing public services and infrastructure despite challenges posed by the COVID-19 pandemic. Key highlights included investments in sustainable urban development and initiatives to support local businesses and community well-being.","key":3,"index":3}],"schema":[{"key":"0","name":"municipality","type":"string"},{"key":"1","name":"year","type":"string"},{"key":"2","name":"ai_response","type":"string"},{"key":"3","name":"generate_response_error","type":"StructType(StructField(response,StringType,true),StructField(status,StructType(StructField(protocolVersion,StructType(StructField(protocol,StringType,true),StructField(major,IntegerType,false),StructField(minor,IntegerType,false)),true),StructField(statusCode,IntegerType,false),StructField(reasonPhrase,StringType,true)),true))"}],"truncated":false},"isSummary":false,"language":"scala","wranglerEntryContext":{"candidateVariableNames":["responses"],"dataframeType":"pyspark"}},"persist_state":{"view":{"type":"details","tableOptions":{},"chartOptions":{"chartType":"bar","categoryFieldKeys":[],"seriesFieldKeys":[],"aggregationType":"sum","isStacked":false,"binsNumber":10,"wordFrequency":"-1"},"viewOptionsGroup":[{"tabItems":[{"type":"table","name":"Table","key":"0","options":{}}]}]}}}}},"dependencies":{"lakehouse":null}},"nbformat":4,"nbformat_minor":5}