test: add ai bot of python language

.github/workflows/ui-test.yml

@@ -245,6 +245,11 @@ jobs:
         with:
           dotnet-version: 6.0.x
 
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
       - name: Install function core tool (ubuntu)
         if: matrix.os == 'ubuntu-latest'
         run: |
@@ -445,6 +450,11 @@ jobs:
         run: |
           npm run build
 
+      - name: Install python extension
+        working-directory: packages/tests
+        run: |
+          npx extest install-from-marketplace --storage .test-resources --extensions_dir .test-resources --type insider ms-python.python
+
       - name: Install docker extension
         if: contains(matrix.test-case,  'docker')
         working-directory: packages/tests

packages/tests/scripts/pvt.json

@@ -63,7 +63,8 @@
       "localdebug-aichat-bot",
       "localdebug-aichat-bot-ts",
       "localdebug-aiassistant-bot",
-      "localdebug-aiassistant-bot-ts"
+      "localdebug-aiassistant-bot-ts",
+      "remotedebug-aichat-bot-py-win-only"
     ],
     "node-20": [
       "localdebug-obo-tab"
@@ -111,7 +112,8 @@
       "localdebug-link-unfurling",
       "localdebug-link-unfurling-ts",
       "localdebug-msg-newapi",
-      "localdebug-msg-newapi-ts"
+      "localdebug-msg-newapi-ts",
+      "localdebug-aichat-bot-py"
     ],
     "node-20": [
       "localdebug-obo-tab"

packages/tests/src/ui-test/localdebug/localdebug-aichat-bot-py.test.ts

@@ -0,0 +1,89 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+/**
+ * @author Helly Zhang <v-helzha@microsoft.com>
+ */
+import * as path from "path";
+import {
+  createEnvironmentWithPython,
+  startDebugging,
+  waitForTerminal,
+} from "../../utils/vscodeOperation";
+import {
+  initPage,
+  validateWelcomeAndReplyBot,
+} from "../../utils/playwrightOperation";
+import { LocalDebugTestContext } from "./localdebugContext";
+import {
+  Timeout,
+  LocalDebugTaskLabel,
+  DebugItemSelect,
+  ValidationContent,
+  LocalDebugTaskLabel2,
+} from "../../utils/constants";
+import { Env } from "../../utils/env";
+import { it } from "../../utils/it";
+import { editDotEnvFile, validateFileExist } from "../../utils/commonUtils";
+
+describe("Local Debug Tests", function () {
+  this.timeout(Timeout.testCase);
+  let localDebugTestContext: LocalDebugTestContext;
+
+  beforeEach(async function () {
+    // ensure workbench is ready
+    this.timeout(Timeout.prepareTestCase);
+    localDebugTestContext = new LocalDebugTestContext("aichat", "python");
+    await localDebugTestContext.before();
+  });
+
+  afterEach(async function () {
+    this.timeout(Timeout.finishTestCase);
+    await localDebugTestContext.after(false, true);
+  });
+
+  it(
+    "[auto] [Python] Local debug AI chat bot",
+    {
+      testPlanCaseId: 27178071,
+      author: "v-helzha@microsoft.com",
+    },
+    async function () {
+      const projectPath = path.resolve(
+        localDebugTestContext.testRootFolder,
+        localDebugTestContext.appName
+      );
+      validateFileExist(projectPath, "src/app.py");
+      const envPath = path.resolve(projectPath, "env", ".env.local.user");
+      editDotEnvFile(envPath, "SECRET_AZURE_OPENAI_API_KEY", "fake");
+      editDotEnvFile(envPath, "AZURE_OPENAI_ENDPOINT", "https://test.com");
+      editDotEnvFile(envPath, "AZURE_OPENAI_MODEL_DEPLOYMENT_NAME", "fake");
+
+      await createEnvironmentWithPython();
+
+      await startDebugging(DebugItemSelect.DebugInTeamsUsingChrome);
+
+      await waitForTerminal(LocalDebugTaskLabel.StartLocalTunnel);
+      await waitForTerminal(
+        LocalDebugTaskLabel2.PythonDebugConsole,
+        "Running on http://localhost:3978"
+      );
+
+      const teamsAppId = await localDebugTestContext.getTeamsAppId();
+      const page = await initPage(
+        localDebugTestContext.context!,
+        teamsAppId,
+        Env.username,
+        Env.password
+      );
+      await localDebugTestContext.validateLocalStateForBot();
+      await validateWelcomeAndReplyBot(page, {
+        hasWelcomeMessage: false,
+        hasCommandReplyValidation: true,
+        botCommand: "helloWorld",
+        expectedWelcomeMessage: ValidationContent.AiChatBotWelcomeInstruction,
+        expectedReplyMessage: ValidationContent.AiBotErrorMessage,
+      });
+    }
+  );
+});

packages/tests/src/ui-test/localdebug/localdebugContext.ts

@@ -37,12 +37,12 @@ export type LocalDebugTestName =
 
 export class LocalDebugTestContext extends TestContext {
   public testName: LocalDebugTestName;
-  public lang: "javascript" | "typescript" = "javascript";
+  public lang: "javascript" | "typescript" | "python" = "javascript";
   needMigrate: boolean | undefined;
 
   constructor(
     testName: LocalDebugTestName,
-    lang: "javascript" | "typescript" = "javascript",
+    lang: "javascript" | "typescript" | "python" = "javascript",
     needMigrate?: boolean
   ) {
     super(testName);
@@ -55,7 +55,7 @@ export class LocalDebugTestContext extends TestContext {
     await super.before();
     await this.createProject();
     await VSBrowser.instance.driver.sleep(30000);
-    await this.disableDebugConsole();
+    // await this.disableDebugConsole();
     const testFolder = path.resolve(this.testRootFolder, this.appName);
     await openExistingProject(testFolder);
   }

packages/tests/src/ui-test/remotedebug/remotedebug-aichat-bot-py-win-only.test.ts

@@ -0,0 +1,114 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+/**
+ * @author Helly Zhang <v-helzha@microsoft.com>
+ */
+import * as path from "path";
+import { VSBrowser } from "vscode-extension-tester";
+import { Timeout, ValidationContent } from "../../utils/constants";
+import {
+  RemoteDebugTestContext,
+  deployProject,
+  provisionProject,
+} from "./remotedebugContext";
+import {
+  execCommandIfExist,
+  createNewProject,
+} from "../../utils/vscodeOperation";
+import {
+  initPage,
+  validateWelcomeAndReplyBot,
+} from "../../utils/playwrightOperation";
+import { Env } from "../../utils/env";
+import { it } from "../../utils/it";
+import { editDotEnvFile, validateFileExist } from "../../utils/commonUtils";
+import { RetryHandler } from "../../utils/retryHandler";
+
+describe("Remote debug Tests", function () {
+  this.timeout(Timeout.testAzureCase);
+  let remoteDebugTestContext: RemoteDebugTestContext;
+  let testRootFolder: string;
+  let appName: string;
+  const appNameCopySuffix = "copy";
+  let newAppFolderName: string;
+  let projectPath: string;
+
+  beforeEach(async function () {
+    // ensure workbench is ready
+    this.timeout(Timeout.prepareTestCase);
+    remoteDebugTestContext = new RemoteDebugTestContext("aichat");
+    testRootFolder = remoteDebugTestContext.testRootFolder;
+    appName = remoteDebugTestContext.appName;
+    newAppFolderName = appName + appNameCopySuffix;
+    projectPath = path.resolve(testRootFolder, newAppFolderName);
+    await remoteDebugTestContext.before();
+  });
+
+  afterEach(async function () {
+    this.timeout(Timeout.finishAzureTestCase);
+    await remoteDebugTestContext.after();
+
+    //Close the folder and cleanup local sample project
+    await execCommandIfExist("Workspaces: Close Workspace", Timeout.webView);
+    console.log(`[Successfully] start to clean up for ${projectPath}`);
+    await remoteDebugTestContext.cleanUp(
+      appName,
+      projectPath,
+      false,
+      true,
+      false
+    );
+  });
+
+  it(
+    "[auto][Python] Remote debug for ai chat bot project Tests",
+    {
+      testPlanCaseId: 27178027,
+      author: "v-helzha@microsoft.com",
+    },
+    async function () {
+      const driver = VSBrowser.instance.driver;
+      await createNewProject("aichat", appName, "Python");
+      validateFileExist(projectPath, "src/app.py");
+      const envPath = path.resolve(projectPath, "env", ".env.dev.user");
+      editDotEnvFile(envPath, "SECRET_AZURE_OPENAI_API_KEY", "fake");
+      editDotEnvFile(envPath, "AZURE_OPENAI_ENDPOINT", "https://test.com");
+      editDotEnvFile(envPath, "AZURE_OPENAI_MODEL_DEPLOYMENT_NAME", "fake");
+      await provisionProject(appName, projectPath);
+      await deployProject(projectPath, Timeout.botDeploy);
+      const teamsAppId = await remoteDebugTestContext.getTeamsAppId(
+        projectPath
+      );
+      const page = await initPage(
+        remoteDebugTestContext.context!,
+        teamsAppId,
+        Env.username,
+        Env.password
+      );
+      await driver.sleep(Timeout.longTimeWait);
+      try {
+        await validateWelcomeAndReplyBot(page, {
+          hasWelcomeMessage: false,
+          hasCommandReplyValidation: true,
+          botCommand: "helloWorld",
+          expectedWelcomeMessage: ValidationContent.AiChatBotWelcomeInstruction,
+          expectedReplyMessage: ValidationContent.AiBotErrorMessage,
+        });
+      } catch {
+        await RetryHandler.retry(async () => {
+          await deployProject(projectPath, Timeout.botDeploy);
+          await driver.sleep(Timeout.longTimeWait);
+          await validateWelcomeAndReplyBot(page, {
+            hasWelcomeMessage: false,
+            hasCommandReplyValidation: true,
+            botCommand: "helloWorld",
+            expectedWelcomeMessage:
+              ValidationContent.AiChatBotWelcomeInstruction,
+            expectedReplyMessage: ValidationContent.AiBotErrorMessage,
+          });
+        }, 2);
+      }
+    }
+  );
+});

packages/tests/src/utils/constants.ts

@@ -407,6 +407,7 @@ export class LocalDebugTaskResult {
 
 export enum LocalDebugTaskLabel2 {
   StartBot2 = "Start Bot",
+  PythonDebugConsole = "Python Debug Console",
 }
 
 export enum LocalDebugError {

packages/tests/src/utils/vscodeOperation.ts

@@ -529,7 +529,7 @@ async function setInputTextWsl(
 export async function createNewProject(
   option: OptionType,
   appName: string,
-  lang?: "JavaScript" | "TypeScript",
+  lang?: "JavaScript" | "TypeScript" | "Python",
   testRootFolder?: string,
   appNameCopySuffix = "copy"
 ): Promise<void> {
@@ -1310,3 +1310,20 @@ export async function getOutputLogs(): Promise<string | undefined> {
   }
   return;
 }
+
+export async function createEnvironmentWithPython() {
+  await execCommandIfExist("Python: Create Environment...", Timeout.webView);
+  const input = await InputBox.create();
+  const driver = VSBrowser.instance.driver;
+  await input.selectQuickPick("Venv");
+  await driver.sleep(Timeout.input);
+  await input.selectQuickPick("Python 3.11");
+  await driver.sleep(Timeout.input);
+  await driver.findElement(By.className("quick-input-check-all")).click();
+  await input.confirm();
+  await driver.sleep(Timeout.longTimeWait);
+  await getNotification(
+    "The following environment is selected",
+    Timeout.shortTimeWait
+  );
+}