Remove setup.sh in favor of skyvern CLI (#4737)

tests/smoke_tests/data/workable_yes_or_no/navigation_payload.json

@@ -0,0 +1 @@
+{"name":"Minni Doc","first_name":"Minni","last_name":"Doc","email":"mezname11@yahoo.com","phone_number":"4156804201","portfolio_link":"https://www.amazon.science/author/minidoc","resume_link":"https://test.google.com/resume.pdf","BirthDate":"5/20/1990","Pronouns":"He/him/his","Location":"1 Warriors Way, San Francisco CA 94158","Gender_Identity":"Male","Sexual_Orientation":"Straight","Ethnicity":"Asian/East Asian","Veteran_Status":"N","Disability_Status":"N","Education_Level":"PhD/Doctor","School_Attended":"Stanford University","First_generation_in_college":"Yes","Authorized_to_work_in_US":"Yes","Will you now, or in the future, require sponsorship for employment visa status (e.g. H-1B visa status)?":"No","Current_Employer":"Lyft","Current_Role":"Applied Scientist","LinkedIn":"https://www.linkedin.com/in/xinchi-chen-b1997bb1/","language proficiency":"I am proficient in English and Chinese (Mandarin)","work_experience_1":"company:Amazon AWS, title: Applied Scientist, experience: Lead a team of data scientists, which focus on ML, IR and NLP. We're the core team behind AWS enterprise knowledge base tooling and large language model. QA benchmarking as an example is one of the focus I have here.","work_experience_2":"company:The University of Edingburgh, title: Research Associate, experience: Focused on NLP and deep learning research. Specialized in Capturing argument interaction in semantic role labeling with capsule networks. I also did lots of research on convolutional interaction network for natural language inferences.","work_experience_3":"company:Microsoft, title: Research Intern, experience: Attention-Fused Deep Matching Network for Natural Language Inference","Q01 - What made you go into Data Science / Research in NLP?":"NLP is the perfect combination of my creative and analytical skills. I really love solving hard research problems which could end up benefiting real-world applications that will enhance people's lives. It's such an interesting field that is constantly evolving and I knew I had found my passion when I discovered NLP and data research work.","Q02 - What is your experience with working with developers on a team?":"I have experience creating comprehensive docs for developers. These docs include precise measurements, annotations, and specifics from researches we've done.","Q03 - What is your favorite aspect of software / data science?":"What I love most about software and data science is the fusion of creativity and problem-solving. It allows me to combine my techinical instincts with analytical thinking to craft solutions that not only solves hard problems but benefit people well. I enjoy seeing a product come to life and knowing I had a part in that!","Q04 - Please select your years of experience":"0-5 Years","Q05 - Tell us about your experience and how it relates to this position.":"I'm currently leading a team of research scientists in AWS. To summarize my key skills that relate to this position: Proficiency in designing and implementing LLM systems. Proficiency in leading a scientists team.","Q06 - What are your weaknesses as scientist / developer?":"One of my current areas for improvement lies in my familiarity with certain used tools in the industry, as I haven't worked outside of AWS in the industry. Most of the tools I've been using are built within Amazon AWS","Q07 - Have you worked remotely before?":"Yes","Q08 - Are you comfortable meeting often or even daily for meeting that require you to be on video?":"Yes","Q09 - What are your career goals?":"I envision a lifelong career in data, research and engineering. Always growing and expanding my skillset as the field continues to evolve. I'd like to eventually choose a niche to focus on. I would also like to take on DS lead or manager role eventually to use my leadership skills to drive impact on a team!","expected salary":"I am currently interviewing for roles ranging 180,000 to 250,000."}

tests/smoke_tests/test_prompts.py

@@ -0,0 +1,217 @@
+import pytest
+from dotenv import load_dotenv
+
+from skyvern.forge import app  # noqa: F401
+from skyvern.forge.forge_app_initializer import start_forge_app
+from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
+
+load_dotenv()
+
+
+@pytest.fixture(scope="module", autouse=True)
+def setup_forge_app():
+    start_forge_app()
+    yield
+
+
+VISA_NAVIGATION_PLAYLOAD = {
+    "login_credentials": {"username": "test@gmail.com", "password": "password"},
+    "info": {
+        "name_of_traveller": "Rajesh Patel",
+        "nationality": "Indian",
+        "citizenship": "Indian",
+        "date_of_birth": "20 Jul 1999",
+        "passport_number": "G9999999",
+        "date_of_issue": "21 Jul 2014",
+        "date_of_expiry": "20 Jul 2024",
+        "place_of_issue": "Jaipur",
+        "national_identity_card": "Aadhar Card",
+        "aadhar_card_number": "300009999999",
+        "name_on_aadhar_card": "Rajesh Patel",
+        "address": "JMR 999, xyz road, Marathalli, Bengaluru, Karnataka, 560106",
+        "relationship_status": "Married",
+        "phone_number": "9999999999",
+        "email": "test@gmail.com",
+        "date_of_departure_from_india": "01/06/2024",
+        "date_of_departure_from_australia": "25/06/2024",
+        "employment_type": "Employed",
+        "company_name": "XYZ Services",
+        "start_date_with_current_employer": "02/02/2023",
+        "organisation_address": "JMR 999, xyz road, Marathalli, Bengaluru, Karnataka, 560106",
+        "special_category_of_entry": "no",
+        "pacific_australia_card": "no",
+        "multiple_citizenships": "no",
+        "other_travel_documentation": "N/A",
+    },
+}
+EXTRACTED_INFORMATION_SCHEMA = """{'additionalProperties': False, 'properties': {'quotes': {'items': {'additionalProperties': False, 'properties': {'coverages': {'items': {'additionalProperties': False, 'properties': {'amount': {'description': "The coverage amount in USD, which can be a single value or a range (e.g., '$300,000' or '$300,000/$300,000').", 'type': 'string'}, 'included': {'description': 'Indicates whether the coverage is included in the policy (true or false).', 'type': 'boolean'}, 'type': {'description': "The type of coverage (e.g., 'Bodily Injury Liability') or deductible name.", 'type': 'string'}}, 'type': 'object'}, 'type': 'array'}, 'premium_amount': {'description': "The total premium amount for the whole quote timeframe in USD, formatted as a string (e.g., '$321.57').", 'type': 'string'}, 'timeframe': {'description': 'The duration of the coverage, typically expressed in months or years.', 'type': 'string'}, 'vehicle_coverages': {'items': {'additionalProperties': False, 'properties': {'collision_deductible': {'description': "The collision deductible amount in USD, which is a single value (e.g., '$500') or null if it is not included", 'type': 'string'}, 'comprehensive_deductible': {'description': "The collision deductible amount in USD, which is a single value (e.g., '$500') or null if it is not included", 'type': 'string'}, 'for_vehicle': {'additionalProperties': False, 'description': 'The vehicle that the collision and comprehensive coverage is for', 'properties': {'make': {'description': 'The make of the vehicle', 'type': 'string'}, 'model': {'description': 'The model of the vehicle', 'type': 'string'}, 'year': {'description': 'The year of the vehicle', 'type': 'string'}}, 'type': 'object'}, 'underinsured_property_damage': {'description': "The underinsured property damage limit for this vehicle, which is a limit and a deductible (e.g., '$25,000/$250 deductible') or null if it is not included", 'type': 'string'}}, 'type': 'object'}, 'type': 'array'}}, 'type': 'object'}, 'type': 'array'}}, 'type': 'object'}"""
+llm_api_handler = LLMAPIHandlerFactory.get_llm_api_handler("OPENAI_GPT5_1")
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip(reason="temporarily disabled: flaky in staging")
+async def test_extract_info_prompt() -> None:
+    """
+    expected_response = {
+        "quotes": [
+            {
+                "premium_amount": "$1,440.11",
+                "timeframe": "6 Month",
+                "coverages": [
+                    {"type": "Bodily Injury Liability", "amount": "$30,000/$60,000", "included": True},
+                    {"type": "Property Damage Liability", "amount": "$25,000", "included": True},
+                    {"type": "Medical Payments", "amount": "Not Included", "included": False},
+                    {"type": "Personal Injury Protection", "amount": "Not Included", "included": False},
+                    {"type": "Uninsured Motorist Bodily Injury", "amount": "$30,000/$60,000", "included": True},
+                    {"type": "Upgraded Accident Forgiveness", "amount": "Not Included", "included": False},
+                ],
+                "vehicle_coverages": [
+                    {
+                        "for_vehicle": {"year": "2015", "make": "Audi", "model": "A8"},
+                        "collision_deductible": "$1,000",
+                        "comprehensive_deductible": "$1,000",
+                        "underinsured_property_damage": "$25,000/$250 deductible",
+                    }
+                ],
+            }
+        ]
+    }
+    """
+    extract_information_prompt = prompt_engine.load_prompt(
+        "extract-information",
+        data_extraction_goal="extract all quote information in JSON format including the premium amount, and the timeframe for the quote",
+        extracted_information_schema=EXTRACTED_INFORMATION_SCHEMA,
+        current_url="https://www.geico.com/quote/",
+    )
+    image_path = "tests/smoke_tests/data/extract_information_screenshot.png"
+    image = open(image_path, "rb").read()
+    parsed_response = await llm_api_handler(
+        prompt=extract_information_prompt, screenshots=[image], step=None, prompt_name="extract-information"
+    )
+
+    assert "quotes" in parsed_response
+    assert len(parsed_response["quotes"]) == 1
+
+    # assertions on quote
+    quote = parsed_response["quotes"][0]
+    expected_quote_attrs = ["premium_amount", "timeframe", "coverages", "vehicle_coverages"]
+    for attr in expected_quote_attrs:
+        assert attr in quote
+    # assert quote["premium_amount"] == "$1,440.11"
+    timeframe_lowercase = quote["timeframe"].lower()
+    assert timeframe_lowercase == "6 month" or timeframe_lowercase == "6 months"
+    assert len(quote["coverages"]) == 6
+
+    # assertions on coverages
+    expected_coverage_types = {
+        "Bodily Injury Liability",
+        "Property Damage Liability",
+        "Medical Payments",
+        "Personal Injury Protection",
+        "Uninsured Motorist Bodily Injury",
+        "Upgraded Accident Forgiveness",
+    }
+    for expected_coverage_type in expected_coverage_types:
+        assert any([expected_coverage_type in coverage["type"] for coverage in quote["coverages"]])
+    for coverage in quote["coverages"]:
+        if "Bodily Injury Liability" in coverage["type"]:
+            assert coverage["included"] is True
+            assert coverage["amount"] == "$30,000/$60,000" or coverage["amount"] == "$30000/$60000"
+        elif "Property Damage Liability" in coverage["type"]:
+            assert coverage["included"] is True
+            assert coverage["amount"] == "$25,000" or coverage["amount"] == "$25000"
+        elif "Medical Payments" in coverage["type"]:
+            assert coverage["included"] is False
+        elif "Personal Injury Protection" in coverage["type"]:
+            assert coverage["included"] is False
+        elif "Uninsured Motorist Bodily Injury" in coverage["type"]:
+            assert coverage["amount"] == "$30,000/$60,000" or coverage["amount"] == "$30000/$60000"
+            assert coverage["included"] is True
+        elif "Upgraded Accident Forgiveness" in coverage["type"]:
+            assert coverage["included"] is False
+
+    # assertions on vehicle
+    assert len(quote["vehicle_coverages"]) == 1
+    vehicle = quote["vehicle_coverages"][0]
+    assert vehicle["for_vehicle"] == {"year": "2015", "make": "Audi", "model": "A8"}
+    assert "$1,000" in vehicle["collision_deductible"] or "$1000" in vehicle["collision_deductible"]
+    assert "$1,000" in vehicle["comprehensive_deductible"] or "$1000" in vehicle["comprehensive_deductible"]
+    assert vehicle["underinsured_property_damage"] == "$25,000/$250 deductible"
+
+
+# @pytest.mark.asyncio
+# async def test_yes_or_no() -> None:
+#     prompt = prompt_engine.load_prompt(
+#         "extract-action",
+#         navigation_goal="Login Into the Portal >> Click On Add New Applicaiton >> Select Visa Category -Visitor> Click on Visa Visitor (600)>> fill out the application. Refer to error banner on the top when there are errors but do not click any of those links. Do not use any link from 'Related Links' or 'Help and Support'. You're done when the application is successfully filled out",
+#         navigation_payload_str=json.dumps(VISA_NAVIGATION_PLAYLOAD),
+#         url="https://online.immi.gov.au/elp/app",
+#         elements=get_complex_element_tree(),
+#         data_extraction_goal=None,
+#         action_history=[],
+#         utc_datetime=datetime.utcnow(),
+#     )
+
+#     screenshot1 = encode_image("tests/smoke_tests/data/complex_context_screenshot1.png")
+#     screenshot2 = encode_image("tests/smoke_tests/data/complex_context_screenshot2.png")
+#     llm_request = build_chat_request(prompt, [screenshot1, screenshot2])
+#     response = await openai_client.client.chat.completions.with_raw_response.create(**llm_request)
+#     chat_completion = response.parse()
+#     parse_response(chat_completion)
+#     assert True
+
+
+# @pytest.mark.asyncio
+# async def test_geico_closest_coverage() -> None:
+#     prompt = prompt_engine.load_prompt(
+#         "extract-action",
+#         navigation_goal="Navigate through the website until you generate an auto insurance quote. Only stay on the\nstarting website, do not navigate to other carrier websites.\nDo not start over, terminate instead.\nIf the only options are going to a different carrier's website or talking to an agent, terminate.\nDo not generate a home insurance quote.\nChoose the following coverage levels\n  * Bodily Injury: $300k/500k\n  * Property Damage: 500k\n  * Underinsured Motorist Bodily Injury: $250k/500k\nChoose the following deductibles for the vehicles:\n  * 2021 Chevrolet Silverado 1500 collision deductible of $2000 and comprehensive deductible of $2000\n  * 2023 GMC Yukon XL collision deductible of $2000 and comprehensive deductible of $2000\n  * 1994 Chevrolet Camaro collision deductible of $2000 and comprehensive deductible of $2000\n  * 2017 Cadillac Escalade ESV collision deductible of $2000 and comprehensive deductible of $2000\nChoose coverages and deductibles that are consistent with the ideal_coverages key.\nIf the coverage level we're looking for is not in the option list, select available coverage level that's closest to what we're looking for. Example 1: if the options are $5,000, $10,000, $15,000, $20,000, $25,000, $50,000, $100,000, and the wanted level is $500,000, select $100,000 as it's the closest number to $500,000. Example 2: if the options are $5,000, $10,000, $15,000, $20,000, $25,000, $50,000, $100,000, and the wanted level is $70,000. 70000-5000=65000, 70000-10000=60000, 70000-15000=55000, 70000-25000, 70000-50000 = 20000, 100000-70000=30000. 20000 is the smallest so $50,000 is the closest coverage, select $50,000\nIf this page contains an auto insurance quote, consider the goal achieved.\n\nIf auto insurance quote amount, which should be number that represents money, is not found,\nthink if there are still actions to take to get the amount. For examples, sometimes\nyou need to recalculate the quote if you have updated the coverages.",
+#         navigation_payload_str=get_file("tests/smoke_tests/data/geico_closest_coverage/navigation_payload.json"),
+#         url="https://www.geico.com",
+#         elements=get_file("tests/smoke_tests/data/geico_closest_coverage/element_tree.json"),
+#         data_extraction_goal='Extract all quote information in JSON format including the premium amount, the timeframe for the quote. The bodily injury, property damage, and underinsured motorist coverages should be returned in the "coverages" key. The collision and comprehensive deductibles should be returned in the "vehicle_coverages" key as they are specific to each vehicle. Also return the quote number.',
+#         action_history=[],
+#         utc_datetime=datetime.utcnow(),
+#     )
+
+#     screenshot1 = encode_image("tests/smoke_tests/data/geico_closest_coverage/geico_closest_coverage_prompt_ss1.png")
+#     screenshot2 = encode_image("tests/smoke_tests/data/geico_closest_coverage/geico_closest_coverage_prompt_ss2.png")
+#     screenshot3 = encode_image("tests/smoke_tests/data/geico_closest_coverage/geico_closest_coverage_prompt_ss3.png")
+#     screenshot4 = encode_image("tests/smoke_tests/data/geico_closest_coverage/geico_closest_coverage_prompt_ss4.png")
+#     screenshot5 = encode_image("tests/smoke_tests/data/geico_closest_coverage/geico_closest_coverage_prompt_ss5.png")
+#     screenshot6 = encode_image("tests/smoke_tests/data/geico_closest_coverage/geico_closest_coverage_prompt_ss6.png")
+#     llm_request = build_chat_request(
+#         prompt,
+#         [screenshot1, screenshot2, screenshot3, screenshot4, screenshot5, screenshot6],
+#     )
+#     response = await openai_client.client.chat.completions.with_raw_response.create(**llm_request)
+#     chat_completion = response.parse()
+#     parse_response(chat_completion)
+#     assert True
+
+
+# @pytest.mark.asyncio
+# async def test_workable_yes_or_no() -> None:
+#     prompt = prompt_engine.load_prompt(
+#         "extract-action",
+#         navigation_goal="Apply for a job. Terminate if the job is not available. Fill out all of the fields as best you can, including optional fields. Be safe when filling out fields where the user didn't specify any details for. Consider the goal achieved when all the relevant fields are completed and the job application is submitted successfully. Job application is submitted successfully when it's indicated on the page. If there are constraints for the job application that the applicant isn't satisfying the requirements for (such as language proficiency, job location), terminate with appropriate reasoning.",
+#         navigation_payload_str=get_file("tests/smoke_tests/data/workable_yes_or_no/navigation_payload.json"),
+#         url="https://apply.workable.com/employer-direct-healthcare/j/037FD88783/apply/",
+#         elements=get_file("tests/smoke_tests/data/workable_yes_or_no/element_tree.json"),
+#         action_history=[],
+#         utc_datetime=datetime.utcnow(),
+#     )
+
+#     screenshot1 = encode_image("tests/smoke_tests/data/workable_yes_or_no/screenshot1.png")
+#     screenshot2 = encode_image("tests/smoke_tests/data/workable_yes_or_no/screenshot2.png")
+#     screenshot3 = encode_image("tests/smoke_tests/data/workable_yes_or_no/screenshot3.png")
+#     screenshot4 = encode_image("tests/smoke_tests/data/workable_yes_or_no/screenshot4.png")
+#     llm_request = build_chat_request(
+#         prompt,
+#         [screenshot1, screenshot2, screenshot3, screenshot4],
+#     )
+#     response = await openai_client.client.chat.completions.with_raw_response.create(**llm_request)
+#     chat_completion = response.parse()
+#     parse_response(chat_completion)
+#     assert True