Support for using access token when running UI test on CI (#99)

.github/workflows/cake-build.yml

@@ -35,6 +35,8 @@ jobs:
         dotnet cake
 
     - name: Tests
+      env: 
+        Access_Token_UI_Tests: ${{ secrets.SRC_ACCESS_TOKEN_DOTCOM }}
       run: |
         cd src
         dotnet cake --target Tests

src/Cody.Core/Infrastructure/WebViewsManager.cs

@@ -131,7 +131,7 @@ await _agentService.ResolveWebviewView(new ResolveWebviewViewParams
         private async Task WaitForAgentInitialization()
         {
             var startTime = DateTime.Now;
-            while (!_agentProxy.IsInitialized)
+            while (!_agentProxy.IsInitialized || _agentService == null)
             {
                 _logger.Debug("Waiting for Agent initialization ...");
                 await Task.Delay(TimeSpan.FromSeconds(1));
@@ -146,6 +146,8 @@ private async Task WaitForAgentInitialization()
                     throw new Exception(message);
                 }
             }
+
+            _logger.Debug($"IsInitialized:{_agentProxy.IsInitialized} AgentService:{_agentService}");
         }
 
         private void OnRegisterWebViewRequestHandler(object sender, string viewId)

src/Cody.VisualStudio.Tests/ChatLoggedBasicTests.cs

@@ -2,53 +2,24 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
+using EnvDTE;
 using Xunit;
 using Xunit.Abstractions;
 
 namespace Cody.VisualStudio.Tests
 {
-    public class ChatLoggedBasicTests : PlaywrightInitializationTests
+    public class ChatLoggedBasicTests : PlaywrightTestsBase
     {
         public ChatLoggedBasicTests(ITestOutputHelper output) : base(output)
         {
         }
 
-        //[VsFact(Version = VsVersion.VS2022)]
-        public async Task Loads_Properly_InLoggedState()
-        {
-            // given
-            var codyPackage = await GetPackageAsync();
-            var settingsService = codyPackage.UserSettingsService;
-            var accessToken = codyPackage.UserSettingsService.AccessToken;
-
-            var text = "Cody Free or Cody Pro";
-            IReadOnlyList<string> textContents;
-            try
-            {
-                await WaitForPlaywrightAsync();
-                codyPackage.UserSettingsService.AccessToken += "INVALID";
-                await Task.Delay(TimeSpan.FromMilliseconds(500)); // wait for the Chat to response
-
-                // when
-
-                var getStarted = Page.GetByText(text);
-                textContents = await getStarted.AllTextContentsAsync();
-            }
-            finally
-            {
-                settingsService.AccessToken = accessToken; // make it valid
-            }
-
-            // then
-            Assert.Equal(text, textContents.First());
-        }
-
-        //[VsFact(Version = VsVersion.VS2022)]
+        [VsFact(Version = VsVersion.VS2022)]
         public async Task Solution_Name_Is_Added_To_Chat_Input()
         {
             // given
-            OpenSolution(SolutionsPaths.GetConsoleApp1File("ConsoleApp1.sln"));
             await WaitForPlaywrightAsync();
+            await OpenSolution(SolutionsPaths.GetConsoleApp1File("ConsoleApp1.sln"));
 
             // when
             var tags = await GetChatContextTags();
@@ -57,12 +28,12 @@ public async Task Solution_Name_Is_Added_To_Chat_Input()
             Assert.Equal("ConsoleApp1", tags.Last().Name);
         }
 
-        //[VsFact(Version = VsVersion.VS2022)]
+        [VsFact(Version = VsVersion.VS2022)]
         public async Task Active_File_Name_And_Line_Selection_Is_Showing_In_Chat_Input()
         {
             // given
-            OpenSolution(SolutionsPaths.GetConsoleApp1File("ConsoleApp1.sln"));
             await WaitForPlaywrightAsync();
+            await OpenSolution(SolutionsPaths.GetConsoleApp1File("ConsoleApp1.sln"));
 
             // when
             const int startLine = 3; const int endLine = 5;
@@ -77,8 +48,8 @@ public async Task Active_File_Name_And_Line_Selection_Is_Showing_In_Chat_Input()
             Assert.Equal(endLine, secondTag.EndLine);
         }
 
-        //[VsFact(Version = VsVersion.VS2022)]
-        public async Task Can_you_close_and_reopen_chat_tool_window()
+        [VsFact(Version = VsVersion.VS2022)]
+        public async Task Can_Chat_Tool_Window_Be_Closed_And_Opened_Again()
         {
             await WaitForPlaywrightAsync();
 
@@ -92,7 +63,7 @@ public async Task Can_you_close_and_reopen_chat_tool_window()
         }
 
         //[VsFact(Version = VsVersion.VS2022)]
-        public async Task Does_chat_history_show_up_after_you_have_submitting_a_chat_close_and_reopen_window()
+        public async Task Entered_Prompt_Show_Up_In_Today_History()
         {
             var num = new Random().Next();
             var prompt = $"How to create const with value {num}?";

src/Cody.VisualStudio.Tests/ChatNotLoggedStateTests.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Threading.Tasks;
 using Xunit;
@@ -17,27 +18,26 @@ public ChatNotLoggedStateTests(ITestOutputHelper output) : base(output)
         public async Task Loads_Properly_InNotLoggedState()
         {
             // given
-            var codyPackage = await GetPackageAsync();
-            var settingsService = codyPackage.UserSettingsService;
-            var accessToken = codyPackage.UserSettingsService.AccessToken;
 
             var text = "Cody Free or Cody Pro";
             IReadOnlyList<string> textContents;
+            string accessToken = null;
             try
             {
                 await WaitForPlaywrightAsync();
-                codyPackage.UserSettingsService.AccessToken += "INVALID";
-                await Task.Delay(TimeSpan.FromMilliseconds(500)); // wait for the Chat to response
 
-                // when
+                accessToken = await GetAccessToken();
+                if (accessToken != null)
+                    await SetAccessToken("INVALID");
 
+                // when
                 var getStarted = Page.GetByText(text);
                 textContents = await getStarted.AllTextContentsAsync();
             }
             finally
             {
                 if (accessToken != null)
-                    settingsService.AccessToken = accessToken; // make it valid
+                    await SetAccessToken(accessToken); // make it valid
             }
 
             // then

src/Cody.VisualStudio.Tests/PlaywrightTestsBase.cs

@@ -49,6 +49,13 @@ private async Task InitializeAsync()
                 await WaitForChat();
                 WriteLog("Chat initialized and loaded.");
 
+                var accessToken = Environment.GetEnvironmentVariable("Access_Token_UI_Tests");
+                if (accessToken != null)
+                {
+                    WriteLog("Using Access Token.");
+                    CodyPackage.UserSettingsService.AccessToken = accessToken;
+                }
+
                 Playwright = await Microsoft.Playwright.Playwright.CreateAsync();
                 WriteLog("Playwright created.");
 
@@ -59,6 +66,10 @@ private async Task InitializeAsync()
                 Context = Browser.Contexts[0];
                 Page = Context.Pages[0];
 
+                //var playwrightTimeout = (float)TimeSpan.FromMinutes(3).TotalMilliseconds;
+                //Page.SetDefaultNavigationTimeout(playwrightTimeout);
+                //Page.SetDefaultTimeout(playwrightTimeout);
+
                 _isInitialized = true;
             }
             finally
@@ -99,6 +110,21 @@ await OnUIThread(() =>
             });
         }
 
+        protected async Task<string> GetAccessToken()
+        {
+            await WaitForPlaywrightAsync();
+
+            var accessToken = CodyPackage.UserSettingsService.AccessToken;
+
+            return accessToken;
+        }
+
+        protected async Task SetAccessToken(string accessToken)
+        {
+            CodyPackage.UserSettingsService.AccessToken = accessToken;
+            await Task.Delay(TimeSpan.FromMilliseconds(2000)); // wait for the Chat to response
+        }
+
         protected async Task ShowChatTab() => await Page.GetByTestId("tab-chat").ClickAsync();
 
         protected async Task ShowHistoryTab() => await Page.GetByTestId("tab-history").ClickAsync();
@@ -124,8 +150,7 @@ protected async Task EnterChatTextAndSend(string prompt)
 
         protected async Task<string[]> GetTodayChatHistory()
         {
-            var todaySection = await Page.QuerySelectorAsync("div[id='radix-:r11:']") ??
-                await Page.QuerySelectorAsync("div[id='radix-:r1b:']");
+            var todaySection = await Page.QuerySelectorAsync("div[id='history-today-content']");
 
             return (await todaySection.QuerySelectorAllAsync("button span"))
                 .Select(async x => await x.TextContentAsync())
@@ -138,6 +163,8 @@ protected async Task<IReadOnlyCollection<ContextTag>> GetChatContextTags()
             var tagsList = new List<ContextTag>();
 
             var chatBox = await Page.QuerySelectorAsync("[aria-label='Chat message']");
+            if (chatBox == null) throw new Exception("ChatBox is null. Probably not authenticated.");
+
             var list = await chatBox.QuerySelectorAllAsync("span[data-lexical-decorator='true']");
             foreach (var item in list)
             {

src/Cody.VisualStudio.Tests/Properties/AssemblyInfo.cs

@@ -32,4 +32,4 @@
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
 [assembly: TestFramework("Xunit.VsTestFramework", "VsixTesting.Xunit")]
-[assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly, DisableTestParallelization = true)]
+[assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly, DisableTestParallelization = true, MaxParallelThreads = 1)]

src/Cody.VisualStudio.Tests/TestsBase.cs

@@ -17,7 +17,7 @@ public abstract class TestsBase : ITestLogger
     {
         private readonly ITestOutputHelper _logger;
 
-        protected CodyPackage CodyPackage;
+        protected static CodyPackage CodyPackage;
 
         protected TestsBase(ITestOutputHelper output)
         {
@@ -28,7 +28,14 @@ protected TestsBase(ITestOutputHelper output)
 
         public void WriteLog(string message, string type = "", [CallerMemberName] string callerName = "")
         {
-            _logger.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] [{type}] [{callerName}] [ThreadId:{Thread.CurrentThread.ManagedThreadId}] {message}");
+            try
+            {
+                _logger.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] [{type}] [{callerName}] [ThreadId:{Thread.CurrentThread.ManagedThreadId}] {message}");
+            }
+            catch
+            {
+                // ignored, because of https://github.com/xunit/xunit/issues/2146
+            }
         }
 
         private IVsUIShell _uiShell;
@@ -37,12 +44,19 @@ public void WriteLog(string message, string type = "", [CallerMemberName] string
         private DTE2 _dte;
         protected DTE2 Dte => _dte ?? (_dte = (DTE2)Package.GetGlobalService(typeof(DTE)));
 
-        protected void OpenSolution(string path) => Dte.Solution.Open(path);
+        protected async Task OpenSolution(string path)
+        {
+            Dte.Solution.Open(path);
+
+            await Task.Delay(TimeSpan.FromSeconds(5));
+        }
 
         protected void CloseSolution() => Dte.Solution.Close();
 
         protected async Task OpenDocument(string path, int? selectLineStart = null, int? selectLineEnd = null)
         {
+            WriteLog($"CodyPackage: {CodyPackage} ");
+
             VsShellUtilities.OpenDocument(CodyPackage, path, Guid.Empty, out _, out _, out IVsWindowFrame frame);
             frame.Show();
 
@@ -105,7 +119,7 @@ protected async Task<CodyPackage> GetPackageAsync()
         protected async Task WaitForAsync(Func<bool> condition)
         {
             var startTime = DateTime.Now;
-            var timeout = TimeSpan.FromMinutes(5);
+            var timeout = TimeSpan.FromMinutes(2);
             while (!condition.Invoke())
             {
                 await Task.Delay(TimeSpan.FromSeconds(1));

src/build.cake

@@ -46,7 +46,7 @@ var codyRepo = "https://github.com/sourcegraph/cody.git";
 var nodeBinaryUrl = "https://github.com/sourcegraph/node-binaries/raw/main/v20.12.2/node-win-x64.exe";
 var nodeArmBinaryUrl = "https://github.com/sourcegraph/node-binaries/raw/main/v20.12.2/node-win-arm64.exe";
 
-var codyCommit = "9a7e712ffce1b90bf2627dd528a68a803b7959ee";
+var codyCommit = "503154d7b220be0ebcfff5d58e6348293a98ff63"; // points to https://github.com/sourcegraph/cody/pull/5532 in the main branch
 
 var marketplaceToken = EnvironmentVariable("CODY_VS_MARKETPLACE_RELEASE_TOKEN");
 
@@ -79,14 +79,21 @@ Task("BuildCodyAgent")
 
 	if (!isDevMode)
 	{
-		Information($"--> Checkout '{branchName}' ...");
-		GitCheckout(codyDir, branchName);
 
+        if (DirectoryExists(codyDir))
+        {
+            Information($"--> Switching to branch {branchName} ...");
+            GitCheckout(codyDir, branchName);
+        }
 
-		//GitCheckout(codyDir, codyCommit);
+        Information($"--> git pull ...");
+		GitPull(codyDir, "cake", "[email protected]", "", "", "origin");
+
+		Information($"--> Checkout '{branchName}' ...");
+		GitCheckout(codyDir, branchName);
 
-		Information($"--> git pull ...");
-		GitPull(codyDir, "cake", "[email protected]");
+        Information($"--> Checkout '{branchName}' using commit {codyCommit} ...");
+		GitCheckout(codyDir, codyCommit);
 	}
 
 	Information($"--> Cleaning '{codyAgentDistDir}' ...");