feat(manager/pipenv): Support custom environment variable usage in Pipfile source URLs

lib/modules/manager/pipenv/artifacts.spec.ts

@@ -15,9 +15,10 @@ import type { RepoGlobalConfig } from '../../../config/types';
 import * as docker from '../../../util/exec/docker';
 import type { StatusResult } from '../../../util/git/types';
 import { find as _find } from '../../../util/host-rules';
-import { getPkgReleases as _getPkgReleases } from '../../datasource';
 import * as _datasource from '../../datasource';
+import { getPkgReleases as _getPkgReleases } from '../../datasource';
 import type { UpdateArtifactsConfig } from '../types';
+import { extractEnvironmentVariableName } from './artifacts';
 import type { PipfileLockSchema } from './schema';
 import { updateArtifacts } from '.';
 
@@ -628,7 +629,19 @@ describe('modules/manager/pipenv/artifacts', () => {
     ]);
   });
 
-  it('does not pass private credential environment vars if variable names differ from allowed', async () => {
+  it('extracts correct environment variable from credential placeholder', () => {
+    [
+      ['$USERNAME', 'USERNAME'],
+      ['$', null],
+      ['${USERNAME}', 'USERNAME'],
+      ['${USERNAME:-default}', 'USERNAME'],
+      ['${COMPLEX_NAME_1:-default}', 'COMPLEX_NAME_1'],
+    ].every((testCase) => {
+      expect(extractEnvironmentVariableName(testCase[0])).toEqual(testCase[1]);
+    });
+  });
+
+  it('updates extraEnv if variable names differ from default', async () => {
     fs.ensureCacheDir.mockResolvedValueOnce(pipenvCacheDir);
     fs.ensureCacheDir.mockResolvedValueOnce(pipCacheDir);
     fs.ensureCacheDir.mockResolvedValueOnce(virtualenvsCacheDir);
@@ -641,6 +654,11 @@ describe('modules/manager/pipenv/artifacts', () => {
     );
     fs.readLocalFile.mockResolvedValueOnce('New Pipfile.lock');
 
+    find.mockReturnValueOnce({
+      username: 'usernameOne',
+      password: 'passwordTwo',
+    });
+
     expect(
       await updateArtifacts({
         packageFileName: 'Pipfile',
@@ -666,6 +684,8 @@ describe('modules/manager/pipenv/artifacts', () => {
           env: {
             PIPENV_CACHE_DIR: pipenvCacheDir,
             WORKON_HOME: virtualenvsCacheDir,
+            USERNAME_FOO: 'usernameOne',
+            PAZZWORD: 'passwordTwo',
           },
         },
       },

lib/modules/manager/pipenv/artifacts.ts

@@ -13,6 +13,7 @@
 } from '../../../util/fs';
 import { getRepoStatus } from '../../../util/git';
 import { find } from '../../../util/host-rules';
+import { parseUrl } from '../../../util/url';
 import { PypiDatasource } from '../../datasource/pypi';
 import type {
   UpdateArtifact,
@@ -113,35 +114,58 @@
   return '';
 }
 
-function getMatchingHostRule(url: string): HostRule {
-  return find({ hostType: PypiDatasource.id, url });
+function getMatchingHostRule(url: string): HostRule | null {
+  const parsedUrl = parseUrl(url);
+  if (!parsedUrl) {
+    return null;
+  }
+  parsedUrl.username = '';
+  parsedUrl.password = '';
+  const urlWithoutCredentials = parsedUrl.toString();
+
+  return find({ hostType: PypiDatasource.id, url: urlWithoutCredentials });
 }
 
-async function findPipfileSourceUrlWithCredentials(
+async function findPipfileSourceUrlsWithCredentials(
   pipfileContent: string,
   pipfileName: string,
-): Promise<string | null> {
+): Promise<string[]> {
   const pipfile = await extractPackageFile(pipfileContent, pipfileName);
   if (!pipfile) {
     logger.debug('Error parsing Pipfile');
-    return null;
+    return [];
   }
 
-  const credentialTokens = [
-    '$USERNAME:',
-    // eslint-disable-next-line no-template-curly-in-string
-    '${USERNAME}',
-    '$PASSWORD@',
-    // eslint-disable-next-line no-template-curly-in-string
-    '${PASSWORD}',
-  ];
+  return pipfile.registryUrls?.filter((url) => !!parseUrl(url)?.username) ?? [];
+}
 
-  const sourceWithCredentials = pipfile.registryUrls?.find((url) =>
-    credentialTokens.some((token) => url.includes(token)),
-  );
+export function extractEnvironmentVariableName(
+  credential: string | null,
+): string | null {
+  if (!credential) {
+    return null;
+  }
+  const match = decodeURI(credential).match('([a-zA-Z0-9_]+)');
+  return match?.length ? match[0] : null;
+}
 
-  // Only one source is currently supported
-  return sourceWithCredentials ?? null;
+function addExtraEnvVariable(
+  extraEnv: ExtraEnv<unknown>,
+  environmentVariableName: string,
+  environmentValue: string,
+): void {
+  logger.debug(
+    `Adding ${environmentVariableName} environment variable for pipenv`,
+  );
+  if (
+    extraEnv[environmentVariableName] &&
+    extraEnv[environmentVariableName] !== environmentValue
+  ) {
+    logger.warn(
+      `Possible misconfiguration, ${environmentVariableName} is already set to a different value`,
+    );
+  }
+  extraEnv[environmentVariableName] = environmentValue;
 }
 
 export async function updateArtifacts({
@@ -189,25 +213,43 @@
       ],
     };
 
-    const sourceUrl = await findPipfileSourceUrlWithCredentials(
+    const sourceUrls = await findPipfileSourceUrlsWithCredentials(
       newPipfileContent,
       pipfileName,
     );
-    if (sourceUrl) {
+    sourceUrls.every((sourceUrl) => {
+      const parsedSourceUrl = parseUrl(sourceUrl);
+      if (!parsedSourceUrl) {
+        return;
+      }
       logger.debug({ sourceUrl }, 'Pipfile contains credentials');
       const hostRule = getMatchingHostRule(sourceUrl);
       if (hostRule) {
         logger.debug('Found matching hostRule for Pipfile credentials');
         if (hostRule.username) {
-          logger.debug('Adding USERNAME environment variable for pipenv');
-          extraEnv.USERNAME = hostRule.username;
+          const environmentVariableName =
+            extractEnvironmentVariableName(parsedSourceUrl.username) ??
+            'USERNAME';
+          const environmentValue = hostRule.username;
+          addExtraEnvVariable(
+            extraEnv,
+            environmentVariableName,
+            environmentValue,
+          );
         }
         if (hostRule.password) {
-          logger.debug('Adding PASSWORD environment variable for pipenv');
-          extraEnv.PASSWORD = hostRule.password;
+          const environmentVariableName =
+            extractEnvironmentVariableName(parsedSourceUrl.password) ??
+            'PASSWORD';
+          const environmentValue = hostRule.password;
+          addExtraEnvVariable(
+            extraEnv,
+            environmentVariableName,
+            environmentValue,
+          );
         }
       }
-    }
+    });
     execOptions.extraEnv = extraEnv;
 
     logger.trace({ cmd }, 'pipenv lock command');