Add source folder variable to Python Client (#24980)

Currently the Python client generator generates a "flat layout", where
the generated Python module is directly below the project root. This is
a problem for editable installs as tools like `uv` have to generate a
complicated "finder" to load the modules. Several tools like `mypy` and
`ty` fail to use it as they implement their own "import mechanism"
incompatible with the generated finder.

This can be mitigated by switching to the "src layout", where all files
live in the sub-directory "src". Tools like `uv` then only have to place
the absolute path in a `.pth` file, which also works with `mypy` and
`ty`.

Similar to #12119
but in the reverse add a new configuration option "sourceFolder" to make
the base folder configurable. For backward compatibility it is unset by
default, which generates the previous "flat layout". Set it to "src" to
generate the "src layout".

Link: https://packaging.python.org/en/latest/discussions/src-layout-vs-flat-layout/

Signed-off-by: Philipp Hahn <p.hahn@avm.de>
Co-authored-by: Philipp Hahn <p.hahn@avm.de>

Author: pmhahn

bin/configs/python.yaml

@@ -2,6 +2,7 @@ generatorName: python
 outputDir: samples/openapi3/client/petstore/python
 inputSpec: modules/openapi-generator/src/test/resources/3_0/python/petstore-with-fake-endpoints-models-for-testing.yaml
 templateDir: modules/openapi-generator/src/main/resources/python
+# sourceFolder: "src"
 additionalProperties:
   packageName: petstore_api
   useOneOfDiscriminatorLookup: "true"

docs/generators/python.md

@@ -36,6 +36,7 @@ These options may be applied as additional-properties (cli) or configOptions (pl
 |projectName|python project name in setup.py (e.g. petstore-api).| |null|
 |recursionLimit|Set the recursion limit. If not set, use the system default value.| |null|
 |setEnsureAsciiToFalse|When set to true, add `ensure_ascii=False` in json.dumps when creating the HTTP request body.| |false|
+|sourceFolder|directory for generated python source code; 'src' for _src layout_, unset (or '.') for _flat layout_.| |null|
 |supportHttpxSync|Generate synchronous '_sync' variants of each API method (httpx and httpx2 libraries only). Each '_sync' method simply calls the corresponding async method and waits for its completion, so both synchronous and asynchronous methods are available from the same API class.| |false|
 |useIndependentImplicitClients|Give API instances without an explicit or registered default ApiClient an owned client with a copied Configuration.| |false|
 |useOneOfDiscriminatorLookup|Use the discriminator's mapping in oneOf to speed up the model lookup. IMPORTANT: Validation (e.g. one and only one match in oneOf's schemas) will be skipped.| |false|

modules/openapi-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java

@@ -120,6 +120,7 @@ public class PythonClientCodegen extends AbstractPythonCodegen implements Codege
     @Setter protected boolean useIndependentImplicitClients = false;
     @Setter protected boolean compatibleWithPythonLegacy = false;
 
+    private String sourceFolder;
     private String testFolder;
 
     public PythonClientCodegen() {
@@ -198,6 +199,7 @@ public PythonClientCodegen() {
         modelDocTemplateFiles.put("model_doc.mustache", ".md");
         apiDocTemplateFiles.put("api_doc.mustache", ".md");
 
+        sourceFolder = "";
         testFolder = "test";
 
         // default HIDE_GENERATION_TIMESTAMP to true
@@ -227,6 +229,7 @@ public PythonClientCodegen() {
         cliOptions.add(new CliOption(POETRY1_FALLBACK, "Fallback to formatting pyproject.toml to Poetry 1.x format."));
         cliOptions.add(new CliOption(LAZY_IMPORTS, "Enable lazy imports.").defaultValue(Boolean.FALSE.toString()));
         cliOptions.add(new CliOption(BUILD_SYSTEM, "Build system to use in pyproject.toml (setuptools, hatchling).").defaultValue("setuptools"));
+        cliOptions.add(new CliOption(CodegenConstants.SOURCE_FOLDER, "directory for generated python source code; 'src' for _src layout_, unset (or '.') for _flat layout_."));
         cliOptions.add(CliOption.newBoolean(SUPPORT_HTTPX_SYNC, "Generate synchronous '_sync' variants of each API method (httpx and httpx2 libraries only). " +
                 "Each '_sync' method simply calls the corresponding async method and waits for its completion, " +
                 "so both synchronous and asynchronous methods are available from the same API class.").defaultValue(Boolean.FALSE.toString()));
@@ -296,6 +299,9 @@ public void processOpts() {
         if (additionalProperties.containsKey(CodegenConstants.PACKAGE_VERSION)) {
             setPackageVersion((String) additionalProperties.get(CodegenConstants.PACKAGE_VERSION));
         }
+        if (additionalProperties.containsKey(CodegenConstants.SOURCE_FOLDER)) {
+            this.sourceFolder = (String) additionalProperties.get(CodegenConstants.SOURCE_FOLDER);
+        }
 
         additionalProperties.put(CodegenConstants.PROJECT_NAME, projectName);
         additionalProperties.put(CodegenConstants.PACKAGE_NAME, packageName);
@@ -314,8 +320,8 @@ public void processOpts() {
             // tests in <package>/test
             testFolder = packagePath() + File.separatorChar + testFolder;
             // api/model docs in <package>/docs
-            apiDocPath = packagePath() + "/" + apiDocPath;
-            modelDocPath = packagePath() + "/" + modelDocPath;
+            apiDocPath = packagePath() + File.separatorChar + apiDocPath;
+            modelDocPath = packagePath() + File.separatorChar + modelDocPath;
         }
         // make api and model doc path available in mustache template
         additionalProperties.put("apiDocPath", apiDocPath);
@@ -392,8 +398,9 @@ public void processOpts() {
                     USE_INDEPENDENT_IMPLICIT_CLIENTS, useIndependentImplicitClients);
         }
 
-        String modelPath = packagePath() + File.separatorChar + modelPackage.replace('.', File.separatorChar);
-        String apiPath = packagePath() + File.separatorChar + apiPackage.replace('.', File.separatorChar);
+        String srcPath = sourceFolder.isEmpty() ? packagePath() : (sourceFolder + File.separatorChar + packagePath());
+        String modelPath = srcPath + File.separatorChar + modelPackage.replace('.', File.separatorChar);
+        String apiPath = srcPath + File.separatorChar + apiPackage.replace('.', File.separatorChar);
 
         String readmePath = "README.md";
         String readmeTemplate = "README.mustache";
@@ -416,23 +423,23 @@ public void processOpts() {
             supportingFiles.add(new SupportingFile("gitlab-ci.mustache", "", ".gitlab-ci.yml"));
             supportingFiles.add(new SupportingFile("setup.mustache", "", "setup.py"));
             supportingFiles.add(new SupportingFile("pyproject.mustache", "", "pyproject.toml"));
-            supportingFiles.add(new SupportingFile("py.typed.mustache", packagePath(), "py.typed"));
+            supportingFiles.add(new SupportingFile("py.typed.mustache", srcPath, "py.typed"));
         }
-        supportingFiles.add(new SupportingFile("configuration.mustache", packagePath(), "configuration.py"));
-        supportingFiles.add(new SupportingFile("__init__package.mustache", packagePath(), "__init__.py"));
+        supportingFiles.add(new SupportingFile("configuration.mustache", srcPath, "configuration.py"));
+        supportingFiles.add(new SupportingFile("__init__package.mustache", srcPath, "__init__.py"));
         supportingFiles.add(new SupportingFile("__init__model.mustache", modelPath, "__init__.py"));
         supportingFiles.add(new SupportingFile("__init__api.mustache", apiPath, "__init__.py"));
         // Generate the 'signing.py' module, but only if the 'HTTP signature' security scheme is specified in the OAS.
         Map<String, SecurityScheme> securitySchemeMap = openAPI != null ?
                 (openAPI.getComponents() != null ? openAPI.getComponents().getSecuritySchemes() : null) : null;
         List<CodegenSecurity> authMethods = fromSecurity(securitySchemeMap);
         if (ProcessUtils.hasHttpSignatureMethods(authMethods)) {
-            supportingFiles.add(new SupportingFile("signing.mustache", packagePath(), "signing.py"));
+            supportingFiles.add(new SupportingFile("signing.mustache", srcPath, "signing.py"));
         }
 
         // If the package name consists of dots(openapi.client), then we need to create the directory structure like openapi/client with __init__ files.
         String[] packageNameSplits = packageName.split("\\.");
-        String currentPackagePath = "";
+        String currentPackagePath = sourceFolder.isEmpty() ? "" : (sourceFolder + File.separatorChar);
         for (int i = 0; i < packageNameSplits.length - 1; i++) {
             if (i > 0) {
                 currentPackagePath = currentPackagePath + File.separatorChar;
@@ -441,34 +448,34 @@ public void processOpts() {
             supportingFiles.add(new SupportingFile("__init__.mustache", currentPackagePath, "__init__.py"));
         }
 
-        supportingFiles.add(new SupportingFile("exceptions.mustache", packagePath(), "exceptions.py"));
+        supportingFiles.add(new SupportingFile("exceptions.mustache", srcPath, "exceptions.py"));
 
         if (Boolean.FALSE.equals(excludeTests)) {
             supportingFiles.add(new SupportingFile("__init__.mustache", testFolder, "__init__.py"));
         }
 
-        supportingFiles.add(new SupportingFile("api_client.mustache", packagePath(), "api_client.py"));
-        supportingFiles.add(new SupportingFile("api_response.mustache", packagePath(), "api_response.py"));
+        supportingFiles.add(new SupportingFile("api_client.mustache", srcPath, "api_client.py"));
+        supportingFiles.add(new SupportingFile("api_response.mustache", srcPath, "api_response.py"));
 
         if ("asyncio".equals(getLibrary())) {
-            supportingFiles.add(new SupportingFile("asyncio/rest.mustache", packagePath(), "rest.py"));
+            supportingFiles.add(new SupportingFile("asyncio/rest.mustache", srcPath, "rest.py"));
             additionalProperties.put("async", "true");
             additionalProperties.put("asyncio", "true");
         } else if (isHttpxLibrary()) {
-            supportingFiles.add(new SupportingFile("httpx/rest.mustache", packagePath(), "rest.py"));
+            supportingFiles.add(new SupportingFile("httpx/rest.mustache", srcPath, "rest.py"));
             additionalProperties.put("async", "true");
             additionalProperties.put(getLibrary(), "true");
             additionalProperties.put("httpxLibrary", true);
             additionalProperties.put("httpxModule", getLibrary());
             if (Boolean.parseBoolean(String.valueOf(additionalProperties.get(SUPPORT_HTTPX_SYNC)))) {
                 // generate synchronous '_sync' method variants alongside the async ones
                 additionalProperties.put(SUPPORT_HTTPX_SYNC, true);
-                supportingFiles.add(new SupportingFile("httpx/sync_helper.mustache", packagePath(), "sync_helper.py"));
+                supportingFiles.add(new SupportingFile("httpx/sync_helper.mustache", srcPath, "sync_helper.py"));
             } else {
                 additionalProperties.remove(SUPPORT_HTTPX_SYNC);
             }
         } else {
-            supportingFiles.add(new SupportingFile("rest.mustache", packagePath(), "rest.py"));
+            supportingFiles.add(new SupportingFile("rest.mustache", srcPath, "rest.py"));
         }
 
         // 'supportHttpxSync' only makes sense for the (async) httpx and httpx2 libraries
@@ -545,12 +552,26 @@ public String toApiDocFilename(String name) {
 
     @Override
     public String apiFileFolder() {
-        return outputFolder + File.separatorChar + apiPackage().replace('.', File.separatorChar);
+        StringBuffer sb = new StringBuffer(outputFolder);
+        sb.append(File.separatorChar);
+        if (!sourceFolder.isEmpty()) {
+            sb.append(sourceFolder);
+            sb.append(File.separatorChar);
+        }
+        sb.append(apiPackage().replace('.', File.separatorChar));
+        return sb.toString();
     }
 
     @Override
     public String modelFileFolder() {
-        return outputFolder + File.separatorChar + modelPackage().replace('.', File.separatorChar);
+        StringBuffer sb = new StringBuffer(outputFolder);
+        sb.append(File.separatorChar);
+        if (!sourceFolder.isEmpty()) {
+            sb.append(sourceFolder);
+            sb.append(File.separatorChar);
+        }
+        sb.append(modelPackage().replace('.', File.separatorChar));
+        return sb.toString();
     }
 
     @Override

modules/openapi-generator/src/main/resources/python/pyproject.mustache

@@ -29,7 +29,12 @@ repository = "https://{{{gitHost}}}/{{{gitUserId}}}/{{{gitRepoId}}}"
 {{/poetry1}}
 keywords = ["OpenAPI", "OpenAPI-Generator", "{{{appName}}}"]
 {{#poetry1}}
-include = ["{{packageName}}/py.typed"]
+{{#sourceFolder}}
+packages = [
+  { include = "{{{packageName}}}", from = "{{{sourceFolder}}}" },
+]
+{{/sourceFolder}}
+include = ["{{#sourceFolder}}{{{sourceFolder}}}/{{/sourceFolder}}{{packageName}}/py.typed"]
 
 [tool.poetry.dependencies]
 python = "^3.10"
@@ -122,7 +127,7 @@ extension-pkg-whitelist = "pydantic"
 
 [tool.mypy]
 files = [
-  "{{{packageName}}}",
+  "{{#sourceFolder}}{{{sourceFolder}}}/{{/sourceFolder}}{{{packageName}}}",
   #"test",  # auto-generated tests
   "tests", # hand-written tests
 ]

modules/openapi-generator/src/main/resources/python/setup.mustache

@@ -43,7 +43,10 @@ setup(
     url="{{packageUrl}}",
     keywords=["OpenAPI", "OpenAPI-Generator", "{{{appName}}}"],
     install_requires=REQUIRES,
-    packages=find_packages(exclude=["test", "tests"]),
+{{#sourceFolder}}
+    package_dir={"": "{{{sourceFolder}}}"},
+{{/sourceFolder}}
+    packages=find_packages({{#sourceFolder}}where="{{{sourceFolder}}}", {{/sourceFolder}}exclude=["test", "tests"]),
     include_package_data=True,
     {{#licenseInfo}}license="{{.}}",
     {{/licenseInfo}}long_description_content_type='text/markdown',

modules/openapi-generator/src/test/java/org/openapitools/codegen/python/PythonClientCodegenTest.java

@@ -139,6 +139,7 @@ public void testInitialConfigValues() throws Exception {
                         PythonClientCodegen.USE_INDEPENDENT_IMPLICIT_CLIENTS),
                 Boolean.FALSE);
         Assert.assertEquals(codegen.isHideGenerationTimestamp(), true);
+        Assert.assertNull(codegen.additionalProperties().get(CodegenConstants.SOURCE_FOLDER));
     }
 
     @Test
@@ -1685,4 +1686,25 @@ public void testIndependentImplicitClientsPreserveDefaultsWhenDisabled() throws
                 "def close(self) -> None:");
         TestUtils.assertFileNotContains(rest, "def close(self) -> None:");
     }
+
+    @Test( description = "test src-layout with setuptools" )
+    public void testSrcLayoutSetuptools() throws IOException {
+        PythonClientCodegen codegen = new PythonClientCodegen();
+        codegen.additionalProperties().put(CodegenConstants.SOURCE_FOLDER, "src");
+        final String output = generateFiles(codegen, "src/test/resources/3_0/generic.yaml");
+
+        final Path setup = Paths.get(output, "setup.py");
+        assertFileContains(setup, "package_dir={\"\": \"src\"}", "packages=find_packages(where=\"src\", exclude=");
+    }
+
+    @Test( description = "test src-layout with poetry1" )
+    public void testSrcLayoutPoetry() throws IOException {
+        PythonClientCodegen codegen = new PythonClientCodegen();
+        codegen.additionalProperties().put(CodegenConstants.SOURCE_FOLDER, "src");
+        codegen.additionalProperties().put("poetry1", true);
+        final String output = generateFiles(codegen, "src/test/resources/3_0/generic.yaml");
+
+        final Path pyproject = Paths.get(output, "pyproject.toml");
+        assertFileContains(pyproject, "{ include = \"openapi_client\", from = \"src\" },", "include = [\"src/openapi_client/py.typed\"]", "\"src/openapi_client\",");
+    }
 }