feat(manager/pipenv): Support custom environment variable usage in Pipfile source URLs

lib/modules/manager/pipenv/artifacts.spec.ts

@@ -12,11 +12,18 @@ import {
 } from '../../../../test/util';
 import { GlobalConfig } from '../../../config/global';
 import type { RepoGlobalConfig } from '../../../config/types';
+import { logger } from '../../../logger';
 import * as docker from '../../../util/exec/docker';
+import type { ExtraEnv, Opt } from '../../../util/exec/types';
 import type { StatusResult } from '../../../util/git/types';
 import { find as _find } from '../../../util/host-rules';
 import * as _datasource from '../../datasource';
 import type { UpdateArtifactsConfig } from '../types';
+import {
+  addExtraEnvVariable,
+  extractEnvironmentVariableName,
+  getMatchingHostRule,
+} from './artifacts';
 import type { PipfileLockSchema } from './schema';
 import { updateArtifacts } from '.';
 
@@ -626,7 +633,34 @@ describe('modules/manager/pipenv/artifacts', () => {
     ]);
   });
 
-  it('does not pass private credential environment vars if variable names differ from allowed', async () => {
+  it('returns no host rule on invalid url', () => {
+    expect(getMatchingHostRule('')).toBeNull();
+  });
+
+  it.each`
+    credential                      | result
+    ${'$USERNAME'}                  | ${'USERNAME'}
+    ${'$'}                          | ${null}
+    ${''}                           | ${null}
+    ${'${USERNAME}'}                | ${'USERNAME'}
+    ${'${USERNAME:-default}'}       | ${'USERNAME'}
+    ${'${COMPLEX_NAME_1:-default}'} | ${'COMPLEX_NAME_1'}
+  `(
+    'extracts correct environment variable from credential placeholder',
+    ({ credential, result }) => {
+      expect(extractEnvironmentVariableName(credential)).toEqual(result);
+    },
+  );
+
+  it('warns about duplicate placeholders with different values', () => {
+    const extraEnv: Opt<ExtraEnv> = {
+      FOO: '1',
+    };
+    addExtraEnvVariable(extraEnv, 'FOO', '2');
+    expect(logger.warn).toHaveBeenCalledOnce();
+  });
+
+  it('updates extraEnv if variable names differ from default', async () => {
     fs.ensureCacheDir.mockResolvedValueOnce(pipenvCacheDir);
     fs.ensureCacheDir.mockResolvedValueOnce(pipCacheDir);
     fs.ensureCacheDir.mockResolvedValueOnce(virtualenvsCacheDir);
@@ -639,6 +673,11 @@ describe('modules/manager/pipenv/artifacts', () => {
     );
     fs.readLocalFile.mockResolvedValueOnce('New Pipfile.lock');
 
+    find.mockReturnValueOnce({
+      username: 'usernameOne',
+      password: 'passwordTwo',
+    });
+
     expect(
       await updateArtifacts({
         packageFileName: 'Pipfile',
@@ -664,6 +703,8 @@ describe('modules/manager/pipenv/artifacts', () => {
           env: {
             PIPENV_CACHE_DIR: pipenvCacheDir,
             WORKON_HOME: virtualenvsCacheDir,
+            USERNAME_FOO: 'usernameOne',
+            PAZZWORD: 'passwordTwo',
           },
         },
       },

lib/modules/manager/pipenv/artifacts.ts

@@ -13,6 +13,8 @@ import {
 } from '../../../util/fs';
 import { getRepoStatus } from '../../../util/git';
 import { find } from '../../../util/host-rules';
+import { regEx } from '../../../util/regex';
+import { parseUrl } from '../../../util/url';
 import { PypiDatasource } from '../../datasource/pypi';
 import type {
   UpdateArtifact,
@@ -113,35 +115,103 @@ export function getPipenvConstraint(
   return '';
 }
 
-function getMatchingHostRule(url: string): HostRule {
-  return find({ hostType: PypiDatasource.id, url });
+export function getMatchingHostRule(url: string): HostRule | null {
+  const parsedUrl = parseUrl(url);
+  if (parsedUrl) {
+    parsedUrl.username = '';
+    parsedUrl.password = '';
+    const urlWithoutCredentials = parsedUrl.toString();
+
+    return find({ hostType: PypiDatasource.id, url: urlWithoutCredentials });
+  }
+  return null;
 }
 
-async function findPipfileSourceUrlWithCredentials(
+async function findPipfileSourceUrlsWithCredentials(
   pipfileContent: string,
   pipfileName: string,
-): Promise<string | null> {
+): Promise<URL[]> {
   const pipfile = await extractPackageFile(pipfileContent, pipfileName);
-  if (!pipfile) {
-    logger.debug('Error parsing Pipfile');
-    return null;
-  }
 
-  const credentialTokens = [
-    '$USERNAME:',
-    // eslint-disable-next-line no-template-curly-in-string
-    '${USERNAME}',
-    '$PASSWORD@',
-    // eslint-disable-next-line no-template-curly-in-string
-    '${PASSWORD}',
-  ];
+  return (
+    pipfile?.registryUrls
+      ?.map(parseUrl)
+      .filter(is.urlInstance)
+      .filter((url) => is.nonEmptyStringAndNotWhitespace(url.username)) ?? []
+  );
+}
 
-  const sourceWithCredentials = pipfile.registryUrls?.find((url) =>
-    credentialTokens.some((token) => url.includes(token)),
+/**
+ * This will extract the actual variable name from an environment-placeholder:
+ * ${USERNAME:-defaultvalue} will yield 'USERNAME'
+ */
+export function extractEnvironmentVariableName(
+  credential: string,
+): string | null {
+  const match = regEx('([a-z0-9_]+)', 'i').exec(decodeURI(credential));
+  return match?.length ? match[0] : null;
+}
+
+export function addExtraEnvVariable(
+  extraEnv: ExtraEnv<unknown>,
+  environmentVariableName: string,
+  environmentValue: string,
+): void {
+  logger.trace(
+    `Adding ${environmentVariableName} environment variable for pipenv`,
   );
+  if (
+    extraEnv[environmentVariableName] &&
+    extraEnv[environmentVariableName] !== environmentValue
+  ) {
+    logger.warn(
+      `Possible misconfiguration, ${environmentVariableName} is already set to a different value`,
+    );
+  }
+  extraEnv[environmentVariableName] = environmentValue;
+}
 
-  // Only one source is currently supported
-  return sourceWithCredentials ?? null;
+/**
+ * Pipenv allows configuring source-urls for remote repositories with placeholders for credentials, i.e. http://$USER:$PASS@myprivate.repo
+ * if a matching host rule exists for that repository, we need to set the corresponding variables.
+ * Simply substituting them in the URL is not an option as it would impact the hash for the resulting Pipfile.lock
+ *
+ */
+async function addCredentialsForSourceUrls(
+  newPipfileContent: string,
+  pipfileName: string,
+  extraEnv: ExtraEnv<unknown>,
+): Promise<void> {
+  const sourceUrls = await findPipfileSourceUrlsWithCredentials(
+    newPipfileContent,
+    pipfileName,
+  );
+  for (const parsedSourceUrl of sourceUrls) {
+    logger.trace(`Trying to add credentials for ${parsedSourceUrl.toString()}`);
+    const matchingHostRule = getMatchingHostRule(parsedSourceUrl.toString());
+    if (matchingHostRule) {
+      const usernameVariableName = extractEnvironmentVariableName(
+        parsedSourceUrl.username,
+      );
+      if (matchingHostRule.username && usernameVariableName) {
+        addExtraEnvVariable(
+          extraEnv,
+          usernameVariableName,
+          matchingHostRule.username,
+        );
+      }
+      const passwordVariableName = extractEnvironmentVariableName(
+        parsedSourceUrl.password,
+      );
+      if (matchingHostRule.password && passwordVariableName) {
+        addExtraEnvVariable(
+          extraEnv,
+          passwordVariableName,
+          matchingHostRule.password,
+        );
+      }
+    }
+  }
 }
 
 export async function updateArtifacts({
@@ -188,26 +258,7 @@ export async function updateArtifacts({
         },
       ],
     };
-
-    const sourceUrl = await findPipfileSourceUrlWithCredentials(
-      newPipfileContent,
-      pipfileName,
-    );
-    if (sourceUrl) {
-      logger.debug({ sourceUrl }, 'Pipfile contains credentials');
-      const hostRule = getMatchingHostRule(sourceUrl);
-      if (hostRule) {
-        logger.debug('Found matching hostRule for Pipfile credentials');
-        if (hostRule.username) {
-          logger.debug('Adding USERNAME environment variable for pipenv');
-          extraEnv.USERNAME = hostRule.username;
-        }
-        if (hostRule.password) {
-          logger.debug('Adding PASSWORD environment variable for pipenv');
-          extraEnv.PASSWORD = hostRule.password;
-        }
-      }
-    }
+    await addCredentialsForSourceUrls(newPipfileContent, pipfileName, extraEnv);
     execOptions.extraEnv = extraEnv;
 
     logger.trace({ cmd }, 'pipenv lock command');