diff --git a/.cookiecutter.json b/.cookiecutter.json index fd87fa73..e1061cf4 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -23,7 +23,7 @@ "pull_request_strategy": "update-or-create", "post_actions": [], "draft": false, - "baked_commit_ref": "b23a9ed5a4714810d83670ad47cc182764c6d464", + "baked_commit_ref": "e4d0e9bd14396e92e7cc2699a9afe91ca5946468", "drift_managed_branch": "develop" } } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 412c9963..e63747eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v6" + uses: "networktocode/gh-action-setup-poetry-environment@v7" with: poetry-version: "2.1.3" - name: "Linting: ruff format" @@ -36,7 +36,7 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v6" + uses: "networktocode/gh-action-setup-poetry-environment@v7" with: poetry-version: "2.1.3" - name: "Linting: ruff" @@ -65,7 +65,7 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v6" + uses: "networktocode/gh-action-setup-poetry-environment@v7" with: poetry-version: "2.1.3" poetry-install-options: "--only dev,docs" @@ -79,7 +79,7 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v6" + uses: "networktocode/gh-action-setup-poetry-environment@v7" with: poetry-version: "2.1.3" - name: "Checking: poetry lock file" @@ -92,7 +92,7 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v6" + uses: "networktocode/gh-action-setup-poetry-environment@v7" with: poetry-version: "2.1.3" - name: "Linting: yamllint" @@ -107,14 +107,14 @@ jobs: strategy: fail-fast: true matrix: - python-version: ["3.10", "3.13"] + python-version: ["3.10"] env: INVOKE_NETUTILS_PYTHON_VER: "${{ matrix.python-version }}" steps: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v6" + uses: "networktocode/gh-action-setup-poetry-environment@v7" with: poetry-version: "2.1.3" - name: "Get image version" @@ -143,7 +143,7 @@ jobs: strategy: fail-fast: true matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.14"] runs-on: "ubuntu-latest" env: INVOKE_NETUTILS_PYTHON_VER: "${{ matrix.python-version }}" @@ -151,7 +151,7 @@ jobs: - name: "Check out repository code" uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v6" + uses: "networktocode/gh-action-setup-poetry-environment@v7" with: poetry-version: "2.1.3" - name: "Get image version" @@ -185,7 +185,7 @@ jobs: with: fetch-depth: "0" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v6" + uses: "networktocode/gh-action-setup-poetry-environment@v7" with: poetry-version: "2.1.3" - name: "Check for changelog entry" diff --git a/.github/workflows/prepare_release.yml b/.github/workflows/prepare_release.yml new file mode 100644 index 00000000..1a70ab5b --- /dev/null +++ b/.github/workflows/prepare_release.yml @@ -0,0 +1,202 @@ +--- +name: "Prepare Release" +on: # yamllint disable-line rule:truthy rule:comments + workflow_dispatch: + inputs: + bump_rule: + description: "Select the version bump type" + required: true + default: "patch" + type: "choice" + options: + - "prerelease" + - "patch" + - "minor" + - "major" + target_branch: + description: "Create the release from this branch (default: main)." + required: true + default: "main" + date: + description: "Date of the release YYYY-MM-DD (defaults to today's date in the US Eastern TZ)." + required: false + default: "" + previous_version: + description: "The previous version tag to use for generating release notes (defaults to the latest tag or initial commit if no tags exist)." + required: false + default: "" + +jobs: + prepare-release: + permissions: + contents: "write" + pull-requests: "write" + name: "Prepare Release" + runs-on: "ubuntu-latest" + steps: + - name: "Checkout code" + uses: "actions/checkout@v4" + with: + # If target_branch is 'main', use 'develop' as the source branch. Otherwise, the source and target branch are the same. + ref: "${{ github.event.inputs['target_branch'] == 'main' && 'develop' || github.event.inputs['target_branch'] }}" + fetch-depth: 0 # Fetch all history for git tags + + - name: "Setup environment" + uses: "networktocode/gh-action-setup-poetry-environment@v7" + with: + poetry-version: "2.1.3" + poetry-install-options: "--with dev" + + - name: "Validate Branch and Tags" + run: | + # 1. Verify branch exists + if ! git rev-parse --verify origin/${{ github.event.inputs.target_branch }} > /dev/null 2>&1; then + echo "Error: Branch ${{ github.event.inputs.target_branch }} does not exist." + exit 1 + fi + + # 2. Try to get the previous version tag + # If it fails (no tags), get the hash of the first commit + if [ -n "${{ github.event.inputs.previous_version }}" ]; then + PREV_TAG="${{ github.event.inputs.previous_version }}" + echo "Using user-specified previous version: $PREV_TAG" + echo "PREVIOUS_TAG=$PREV_TAG" >> $GITHUB_ENV + elif PREV_TAG=$(git describe --tags --abbrev=0 2>/dev/null); then + echo "PREVIOUS_TAG=$PREV_TAG" >> $GITHUB_ENV + echo "Found previous tag: $PREV_TAG" + else + # Fallback to the first commit in the repository + FIRST_COMMIT=$(git rev-list --max-parents=0 HEAD) + echo "PREVIOUS_TAG=$FIRST_COMMIT" >> $GITHUB_ENV + echo "No tags found. Falling back to initial commit: $FIRST_COMMIT" + fi + + - name: "Determine New Version" + id: "versioning" + env: + GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + run: | + # Bump the poetry version based on the user input + CURRENT_POETRY_VER=$(poetry version --short) + set +e # continue running if the next command fails + GITHUB_RELEASE_EXISTS=$(gh release view v$CURRENT_POETRY_VER --json publishedAt --jq '.publishedAt') + set -e + # Unconditionally bump the version if the current version is alpha0 or beta0 + if [[ $CURRENT_POETRY_VER == *a0 || $CURRENT_POETRY_VER == *b0 || $CURRENT_POETRY_VER == *rc0 ]]; then + poetry version ${{ github.event.inputs.bump_rule }} + # If the bump rule is prerelease and a release doesn't already exist in github we don't bump the version + + # This is because we've already bumped to the prerelease version after the previous release + elif [[ "${{ github.event.inputs.bump_rule }}" == "prerelease" && "$CURRENT_POETRY_VER" =~ (a|b|rc)[0-9]+$ && -z "$GITHUB_RELEASE_EXISTS" ]]; then + echo "Prerelease v$CURRENT_POETRY_VER does not exist in github, using this version" + else + poetry version ${{ github.event.inputs.bump_rule }} + fi + + # Capture the New version string for use in other steps + NEW_VER=$(poetry version --short) + echo "NEW_VERSION=$NEW_VER" >> $GITHUB_ENV + echo "RELEASE_BRANCH=release/$NEW_VER" >> $GITHUB_ENV + + - name: "Set Date Variable" + run: | + if [ -z "${{ github.event.inputs.date }}" ]; then + RELEASE_DATE=$(TZ=America/New_York date +%Y-%m-%d) + else + RELEASE_DATE="${{ github.event.inputs.date }}" + fi + echo "RELEASE_DATE=$RELEASE_DATE" >> $GITHUB_ENV + + - name: "Create Release Branch" + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + run: | + git config user.name "${{ github.actor }}" + git config user.email "${{ github.actor }}@users.noreply.github.com" + + # Ensure release branch doesn't already exist + if git rev-parse --verify origin/${{ env.RELEASE_BRANCH }} > /dev/null 2>&1; then + echo "Error: Release branch ${{ env.RELEASE_BRANCH }} already exists." + exit 1 + fi + + # Create a new branch for the release + git checkout -b "${{ env.RELEASE_BRANCH }}" + + - name: "Regenerate poetry.lock" + run: "poetry lock --regenerate" + + - name: "Generate Github Release Notes" + env: + GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + run: | + # 1. Get Towncrier Draft + TOWNCRIER_NOTES=$(poetry run towncrier build --version "${{ env.NEW_VERSION }}" --date "${{ env.RELEASE_DATE }}" --draft) + + # 2. Call GitHub API to generate raw notes + RAW_GH_NOTES=$(gh api /repos/${{ github.repository }}/releases/generate-notes \ + -f tag_name="v${{ env.NEW_VERSION }}" \ + -f target_commitish="${{ github.event.inputs['target_branch'] == 'main' && 'develop' || github.event.inputs['target_branch'] }}" \ + -f previous_tag_name="${{ env.PREVIOUS_TAG }}" --jq '.body') + + # 3. Parse usernames (Regex match) + # We use grep to find "@user" patterns, sort, and uniq them + USERNAMES=$(echo "$RAW_GH_NOTES" | grep -oP 'by @\K[a-zA-Z0-9-]+' | sort -u | grep -vE 'dependabot|nautobot-bot|github-actions' || true) + + # 4. Format the Contributors section + CONTRIBUTORS_SECTION="## Contributors" + for user in $USERNAMES; do + CONTRIBUTORS_SECTION="$CONTRIBUTORS_SECTION"$'\n'"* @$user" + done + + # 5. Extract the "Full Changelog" or "New Contributors" part + # Using awk to grab everything from '## New Contributors' or '**Full Changelog**' to the end + GH_FOOTER=$(echo "$RAW_GH_NOTES" | awk '/## New Contributors/ || /\*\*Full Changelog\*\*/ {found=1} found {print}') + if [ -z "$GH_FOOTER" ]; then + GH_FOOTER=$(echo "$RAW_GH_NOTES" | sed -n '/**Full Changelog**/,$p') + fi + + # 6. Combine everything + FINAL_NOTES="$TOWNCRIER_NOTES"$'\n\n'"$CONTRIBUTORS_SECTION"$'\n\n'"$GH_FOOTER" + + # 7. Save to a temporary file to avoid shell argument length limits + echo "$FINAL_NOTES" > ../consolidated_notes.md + + - name: "Generate App Release Notes and update mkdocs.yml" + run: "poetry run inv generate-release-notes --version '${{ env.NEW_VERSION }}' --date '${{ env.RELEASE_DATE }}'" + + - name: "Commit Changes and Push" + run: | + # Add all changes (pyproject.toml, poetry.lock, etc.) + git add . + git commit -m "prepare release v${{ env.NEW_VERSION }}" + git push origin "${{ env.RELEASE_BRANCH }}" + + - name: "Create Pull Request" + env: + GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + run: | + gh pr create \ + --title "Release v${{ env.NEW_VERSION }}" \ + --body-file "../consolidated_notes.md" \ + --base "${{ github.event.inputs.target_branch }}" \ + --head "${{ env.RELEASE_BRANCH }}" + + - name: "Create Draft Release" + env: + GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + run: | + if [[ "${{ github.event.inputs.bump_rule }}" == "prerelease" ]]; then + RELEASE_FLAGS="--prerelease" + elif [[ "${{ github.event.inputs.target_branch }}" == "main" ]]; then + RELEASE_FLAGS="--latest" + else + RELEASE_FLAGS="--latest=false" + fi + + gh release create "v${{ env.NEW_VERSION }}" \ + --draft \ + $RELEASE_FLAGS \ + --title "v${{ env.NEW_VERSION }} - ${{ env.RELEASE_DATE }}" \ + --notes-file "../consolidated_notes.md" \ + --target "${{ github.event.inputs.target_branch }}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a6cd1872..a1455227 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,9 +1,13 @@ --- name: "Release" -on: # yamllint disable-line rule:truthy rule:comments +on: # yamllint disable-line rule:truthy rule:comments release: types: ["published"] +env: + POETRY_VERSION: "2.1.3" + PYTHON_VERSION: "3.12" + jobs: build: name: "Build package with poetry" @@ -12,11 +16,13 @@ jobs: steps: - uses: "actions/checkout@v4" - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v6" + uses: "networktocode/gh-action-setup-poetry-environment@v7" with: - poetry-version: "2.1.3" - python-version: "3.13" + poetry-version: "${{ env.POETRY_VERSION }}" + python-version: "${{ env.PYTHON_VERSION }}" poetry-install-options: "--no-root" + - name: "Build Documentation" + run: "poetry run invoke build-and-check-docs" - name: "Run Poetry Build" run: "poetry build" @@ -48,7 +54,7 @@ jobs: - name: "Upload binaries to release" run: "gh release upload ${{ github.ref_name }} dist/*.{tar.gz,whl}" env: - GH_TOKEN: "${{ secrets.NTC_GITHUB_TOKEN }}" + GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}" publish-pypi: name: "Push Package to PyPI" @@ -56,20 +62,18 @@ jobs: if: "startsWith(github.ref, 'refs/tags/v')" needs: "build" environment: "pypi" - # Steps to publish to PyPI. + permissions: + # IMPORTANT: this permission is mandatory for Trusted Publishing + id-token: "write" steps: - name: "Retrieve built package from cache" uses: "actions/download-artifact@v4" with: name: "distfiles" path: "dist/" + - name: "Publish package distributions to PyPI" uses: "pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e" # v1.13.0 - ## Used for networktocode org since trusted publisher isn't supported for GitHub Plan. - with: - user: "__token__" - password: "${{ secrets.PYPI_API_TOKEN }}" - # End publish to PyPI job. slack-notify: needs: @@ -104,3 +108,70 @@ jobs: } ] } + + create-pr-to-develop: + if: "github.event.release.target_commitish == 'main'" + permissions: + contents: "write" + pull-requests: "write" + name: "${{ github.event.release.target_commitish == 'main' && 'Create a PR from main into develop' || format('Create a PR to merge into {0}', github.event.release.target_commitish) }}" + needs: + - "publish-github" + - "publish-pypi" + runs-on: "ubuntu-latest" + steps: + - name: "Checkout source branch" + uses: "actions/checkout@v4" + with: + ref: "${{ github.event.release.target_commitish }}" + fetch-depth: 0 + + - name: "Setup environment" + uses: "networktocode/gh-action-setup-poetry-environment@v7" + with: + poetry-version: "${{ env.POETRY_VERSION }}" + poetry-install-options: "--no-root" + + - name: "Create release branch and bump version" + id: "branch" + run: | + + git config user.name "${{ github.actor }}" + git config user.email "${{ github.actor }}@users.noreply.github.com" + + SOURCE_BRANCH="${{ github.event.release.target_commitish }}" + if [ "$SOURCE_BRANCH" = "main" ]; then + TARGET_BRANCH="develop" + else + TARGET_BRANCH="$SOURCE_BRANCH" + fi + + TAG_NAME="${{ github.event.release.tag_name }}" + VERSION="${TAG_NAME#v}" + + BRANCH_NAME="release-${VERSION}-to-${TARGET_BRANCH}" + + # Ensure release branch doesn't already exist + if git rev-parse --verify origin/$BRANCH_NAME > /dev/null 2>&1; then + echo "Error: Release branch $BRANCH_NAME already exists." + exit 1 + fi + + git checkout -b "$BRANCH_NAME" + + poetry version prerelease + git add pyproject.toml && git commit -m "Bump version" + git push origin "$BRANCH_NAME" + + echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT + echo "target_branch=$TARGET_BRANCH" >> $GITHUB_OUTPUT + + - name: "Create Pull Request" + env: + GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + run: | + gh pr create \ + --title "Post release ${{ github.event.release.tag_name }} to ${{ steps.branch.outputs.target_branch }}" \ + --body "Please do a merge commit." \ + --base "${{ steps.branch.outputs.target_branch }}" \ + --head "${{ steps.branch.outputs.branch_name }}" diff --git a/LICENSE b/LICENSE index 6ce362fa..9e40eee1 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ Apache Software License 2.0 -Copyright (c) 2021-2025, Network to Code, LLC +Copyright (c) 2021-2026, Network to Code, LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/changes/+main.housekeeping b/changes/+main.housekeeping new file mode 100644 index 00000000..3433adf6 --- /dev/null +++ b/changes/+main.housekeeping @@ -0,0 +1 @@ +Rebaked from the cookie `main`. diff --git a/development/bin/ensure_release_notes.py b/development/bin/ensure_release_notes.py new file mode 100644 index 00000000..f038cc4d --- /dev/null +++ b/development/bin/ensure_release_notes.py @@ -0,0 +1,99 @@ +"""Ensure that release notes exist for a given version. + +This script will do the following: + Ensure a release notes file exists at `docs/admin/release_notes/version_{version}.md`. + Ensure the `mkdocs.yml` file is updated to add the release notes file to the navigation. + Ensure the `pyproject.toml` `tool.towncrier.filename` is updated to reference the release notes file. + +It shouldn't be necessary to run this file manually. It is automatically called by `invoke generate-release-notes`. + +Example: + $ python ensure_release_notes.py --version '1.0.0' +""" + +import argparse + +try: + import tomllib +except ImportError: + import tomli as tomllib + +from pathlib import Path + + +def release_notes_pyproject_toml(version): + """Update the pyproject.toml file to set the towncrier filename for the given version.""" + pyproject_file = Path(__file__).parent.parent.parent / "pyproject.toml" + pyproject_content = pyproject_file.read_text() + pyproject_data = tomllib.loads(pyproject_content) + release_notes_file = f"docs/admin/release_notes/version_{version}.md" + + # Update the towncrier filename + if pyproject_data["tool"]["towncrier"].get("filename", "") != release_notes_file: + pyproject_data["tool"]["towncrier"]["filename"] = release_notes_file + + # Write back the updated content to pyproject.toml + # tomllib is not used to write the file because it is not roundtrippable + new_pyproject_content = [] + in_towncrier_section = False + for line in pyproject_content.splitlines(): + if line.strip() == "[tool.towncrier]": + in_towncrier_section = True + new_pyproject_content.append(line) + continue + if in_towncrier_section: + if line.strip().startswith("filename"): + new_pyproject_content.append(f'filename = "docs/admin/release_notes/version_{version}.md"') + in_towncrier_section = False # Only replace the first occurrence + else: + new_pyproject_content.append(line) + else: + new_pyproject_content.append(line) + + pyproject_file.write_text("\n".join(new_pyproject_content)) + # Add a newline at the end of the file if it doesn't exist + if not pyproject_file.read_text().endswith("\n"): + pyproject_file.write_text(pyproject_file.read_text() + "\n") + # Remind the user to update the release notes file. + print( + f"\033[33mRemember to update the Release Overview section in the release notes file: {release_notes_file}\033[0m" + ) + + +def ensure_release_notes_file(version): + """Ensure that the release notes file for the given version exists and is referenced in mkdocs.yml.""" + release_notes_file = ( + Path(__file__).parent.parent.parent / "docs" / "admin" / "release_notes" / f"version_{version}.md" + ) + if not release_notes_file.exists(): + # Create a new release notes file with a basic template from towncrier_header.txt + towncrier_header = Path(__file__).parent.parent / "towncrier_header.txt" + content = towncrier_header.read_text().format(version=version) + release_notes_file.write_text(content) + + +def ensure_mkdocs_version(version): + """Ensure that mkdocs.yml includes the new release notes file in the navigation.""" + mkdocs_yml_file = Path(__file__).parent.parent.parent / "mkdocs.yml" + mkdocs_yml_content = mkdocs_yml_file.read_text() + release_notes_nav_entry = f' - v{version}: "admin/release_notes/version_{version}.md"\n' + if release_notes_nav_entry in mkdocs_yml_content: + return + + # Add the new release notes entry to the mkdocs.yml content + if "Release Notes:" in mkdocs_yml_content: + mkdocs_yml_content = mkdocs_yml_content.replace( + ' - "admin/release_notes/index.md"\n', + f' - "admin/release_notes/index.md"\n{release_notes_nav_entry}', + ) + + mkdocs_yml_file.write_text(mkdocs_yml_content) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Ensure release notes exist for a given version.") + parser.add_argument("--version", help="The version number (e.g. 2.2)") + args = parser.parse_args() + ensure_release_notes_file(args.version) + ensure_mkdocs_version(args.version) + release_notes_pyproject_toml(args.version) diff --git a/towncrier_template.j2 b/development/towncrier_template.j2 similarity index 66% rename from towncrier_template.j2 rename to development/towncrier_template.j2 index 2c1316f5..5fa06c3b 100644 --- a/towncrier_template.j2 +++ b/development/towncrier_template.j2 @@ -1,13 +1,4 @@ -# v{{ versiondata.version.split(".")[:2] | join(".") }} Release Notes - -This document describes all new features and changes in the release. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## Release Overview - -- Major features or milestones -- Changes to compatibility with Nautobot and/or other apps, libraries etc. - {% if render_title %} ## [v{{ versiondata.version }} ({{ versiondata.date }})](https://github.com/networktocode/netutils/releases/tag/v{{ versiondata.version}}) diff --git a/docs/user/lib_use_cases_acl.md b/docs/user/lib_use_cases_acl.md index 15c33e4d..53c57460 100644 --- a/docs/user/lib_use_cases_acl.md +++ b/docs/user/lib_use_cases_acl.md @@ -230,9 +230,9 @@ Here we can test if a rule is matched via the existing ruleset. We can leverage **Inherit Example** ```python - from netutils.acl import ACLRule + class ExpandAddrGroups(ACLRule): address_groups = {"red": ["white", "blue"], "blue": ["cyan"], "yellow": ["orange"]} addresses = {"white": ["10.1.1.1", "10.2.2.2"], "cyan": ["10.3.3.3"], "orange": ["10.4.4.4"]} diff --git a/docs/user/lib_use_cases_jinja_filters.md b/docs/user/lib_use_cases_jinja_filters.md index 14cd4d14..92352742 100644 --- a/docs/user/lib_use_cases_jinja_filters.md +++ b/docs/user/lib_use_cases_jinja_filters.md @@ -20,15 +20,11 @@ IP Address + 200 = {{ "192.168.0.1" | ip_addition(200) }} Below is a code in the `jinja2_environment.py` folder. ```python - from jinja2.loaders import FileSystemLoader, PackageLoader from jinja2 import Environment, PackageLoader, select_autoescape from netutils.utils import jinja2_convenience_function -env = Environment( - loader=FileSystemLoader("templates"), - autoescape=select_autoescape() -) +env = Environment(loader=FileSystemLoader("templates"), autoescape=select_autoescape()) env.filters.update(jinja2_convenience_function()) @@ -51,6 +47,8 @@ In Ansible, one can add with the following code by adding to a folder called `fi ```python from netutils.utils import jinja2_convenience_function + + class FilterModule(object): def filters(self): return jinja2_convenience_function() @@ -91,10 +89,7 @@ from jinja2.loaders import FileSystemLoader, PackageLoader from jinja2 import Environment, PackageLoader, select_autoescape from netutils.utils import jinja2_convenience_function -env = Environment( - loader=FileSystemLoader("templates"), - autoescape=select_autoescape() -) +env = Environment(loader=FileSystemLoader("templates"), autoescape=select_autoescape()) env.filters.update(jinja2_convenience_function()) @@ -132,10 +127,7 @@ from netutils.utils import jinja2_convenience_function env = Environment(loader=BaseLoader()) env.filters.update(jinja2_convenience_function()) -DATA = { - "device": "USSCAMS07", - "comma_seperated_devices": "NYC-RT01,NYC-RT02,SFO-SW01,SFO-RT01" -} +DATA = {"device": "USSCAMS07", "comma_seperated_devices": "NYC-RT01,NYC-RT02,SFO-SW01,SFO-RT01"} TEMPLATE_STRING = """ {% set device_details = '([A-Z]{2})([A-Z]{2})([A-Z]{3})(\d*)' | regex_match(device) %} diff --git a/docs/user/lib_use_cases_lib_mapper.md b/docs/user/lib_use_cases_lib_mapper.md index 55780b9d..91e56be4 100644 --- a/docs/user/lib_use_cases_lib_mapper.md +++ b/docs/user/lib_use_cases_lib_mapper.md @@ -5,14 +5,13 @@ These dictionaries provide mappings in expected vendor names between Netmiko, NA These dictionaries allow you to keep your Source of Truth platform data consistent and still easily switch between automation libraries. For example, you may be storing your device platform data in Nautobot. In a Nautobot platform, you can store the NAPALM driver needed for that platform. What if you wanted to write a python script to leverage the backup capabilities of pyntc? Here's an example of how you could use the following dictionaries to perform mappings from your stored Nautobot NAPALM driver to the pyntc driver needed for your script. ```python - import pynautobot from netutils.lib_mapper import NAPALM_LIB_MAPPER, PYNTC_LIB_MAPPER_REVERSE from pyntc import ntc_device as NTC # Get device from Nautobot -nautobot = pynautobot.api(url="http://mynautobotinstance.com",token="mytoken") +nautobot = pynautobot.api(url="http://mynautobotinstance.com", token="mytoken") # Get Napalm driver and save for later use. device = nautobot.dcim.devices.get(name="mydevice") @@ -22,11 +21,7 @@ sot_driver = device.platform.napalm_driver # Connect to device via Napalm driver = napalm.get_network_driver(sot_driver) -device = driver( - hostname="device.name", - username="demo", - password="secret" -) +device = driver(hostname="device.name", username="demo", password="secret") # Do Napalm tasks @@ -49,7 +44,7 @@ There is also a dynamically built mapping that gives you all of the libraries gi "cisco_nxos": { "ansible": "cisco.nxos.nxos", "napalm": "nxos", - } + }, } ``` diff --git a/docs/user/lib_use_cases_nist.md b/docs/user/lib_use_cases_nist.md index 4fd080fb..3b5cacb1 100644 --- a/docs/user/lib_use_cases_nist.md +++ b/docs/user/lib_use_cases_nist.md @@ -25,7 +25,6 @@ For this reason, for certain Vendor/OS combinations, a custom URL needs to be bu Here are a few examples showing how to use this in your python code. ```python - from netutils.nist import get_nist_urls # Get NIST URL for the Cisco IOS object diff --git a/docs/user/lib_use_cases_protocol_mappers.md b/docs/user/lib_use_cases_protocol_mappers.md index 7c071de4..fa3459dc 100644 --- a/docs/user/lib_use_cases_protocol_mappers.md +++ b/docs/user/lib_use_cases_protocol_mappers.md @@ -30,9 +30,7 @@ Here are currently available mappers: Here are a few examples showing how you would use these in your python code. ```python - from netutils.protocol_mapper import ( - PROTO_NAME_TO_NUM, PROTO_NUM_TO_NAME, TCP_NAME_TO_NUM, diff --git a/example.invoke.yml b/invoke.example.yml similarity index 100% rename from example.invoke.yml rename to invoke.example.yml diff --git a/poetry.lock b/poetry.lock index 1e066a76..d8452e06 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. [[package]] name = "annotated-types" @@ -1374,14 +1374,14 @@ min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4) ; platform [[package]] name = "mkdocs-autorefs" -version = "1.4.4" +version = "1.2.0" description = "Automatically link across pages in MkDocs." optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" groups = ["docs"] files = [ - {file = "mkdocs_autorefs-1.4.4-py3-none-any.whl", hash = "sha256:834ef5408d827071ad1bc69e0f39704fa34c7fc05bc8e1c72b227dfdc5c76089"}, - {file = "mkdocs_autorefs-1.4.4.tar.gz", hash = "sha256:d54a284f27a7346b9c38f1f852177940c222da508e66edc816a0fa55fc6da197"}, + {file = "mkdocs_autorefs-1.2.0-py3-none-any.whl", hash = "sha256:d588754ae89bd0ced0c70c06f58566a4ee43471eeeee5202427da7de9ef85a2f"}, + {file = "mkdocs_autorefs-1.2.0.tar.gz", hash = "sha256:a86b93abff653521bda71cf3fc5596342b7a23982093915cb74273f67522190f"}, ] [package.dependencies] @@ -2724,30 +2724,30 @@ oldlibyaml = ["ruamel.yaml.clib ; platform_python_implementation == \"CPython\"" [[package]] name = "ruff" -version = "0.5.5" +version = "0.16.4" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.5.5-py3-none-linux_armv6l.whl", hash = "sha256:605d589ec35d1da9213a9d4d7e7a9c761d90bba78fc8790d1c5e65026c1b9eaf"}, - {file = "ruff-0.5.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:00817603822a3e42b80f7c3298c8269e09f889ee94640cd1fc7f9329788d7bf8"}, - {file = "ruff-0.5.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:187a60f555e9f865a2ff2c6984b9afeffa7158ba6e1eab56cb830404c942b0f3"}, - {file = "ruff-0.5.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe26fc46fa8c6e0ae3f47ddccfbb136253c831c3289bba044befe68f467bfb16"}, - {file = "ruff-0.5.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4ad25dd9c5faac95c8e9efb13e15803cd8bbf7f4600645a60ffe17c73f60779b"}, - {file = "ruff-0.5.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f70737c157d7edf749bcb952d13854e8f745cec695a01bdc6e29c29c288fc36e"}, - {file = "ruff-0.5.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:cfd7de17cef6ab559e9f5ab859f0d3296393bc78f69030967ca4d87a541b97a0"}, - {file = "ruff-0.5.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a09b43e02f76ac0145f86a08e045e2ea452066f7ba064fd6b0cdccb486f7c3e7"}, - {file = "ruff-0.5.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d0b856cb19c60cd40198be5d8d4b556228e3dcd545b4f423d1ad812bfdca5884"}, - {file = "ruff-0.5.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3687d002f911e8a5faf977e619a034d159a8373514a587249cc00f211c67a091"}, - {file = "ruff-0.5.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ac9dc814e510436e30d0ba535f435a7f3dc97f895f844f5b3f347ec8c228a523"}, - {file = "ruff-0.5.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:af9bdf6c389b5add40d89b201425b531e0a5cceb3cfdcc69f04d3d531c6be74f"}, - {file = "ruff-0.5.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d40a8533ed545390ef8315b8e25c4bb85739b90bd0f3fe1280a29ae364cc55d8"}, - {file = "ruff-0.5.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:cab904683bf9e2ecbbe9ff235bfe056f0eba754d0168ad5407832928d579e7ab"}, - {file = "ruff-0.5.5-py3-none-win32.whl", hash = "sha256:696f18463b47a94575db635ebb4c178188645636f05e934fdf361b74edf1bb2d"}, - {file = "ruff-0.5.5-py3-none-win_amd64.whl", hash = "sha256:50f36d77f52d4c9c2f1361ccbfbd09099a1b2ea5d2b2222c586ab08885cf3445"}, - {file = "ruff-0.5.5-py3-none-win_arm64.whl", hash = "sha256:3191317d967af701f1b73a31ed5788795936e423b7acce82a2b63e26eb3e89d6"}, - {file = "ruff-0.5.5.tar.gz", hash = "sha256:cc5516bdb4858d972fbc31d246bdb390eab8df1a26e2353be2dbc0c2d7f5421a"}, + {file = "ruff-0.16.4-py3-none-linux_armv6l.whl", hash = "sha256:df4075f71ddac40b9934af60c3ec8a53047dd5a5fdc43224e6e4e8e9a27cb6f7"}, + {file = "ruff-0.16.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0c95538517af68004306b0fb3214ff2f2af67a65092aee77cd9eb86db6656604"}, + {file = "ruff-0.16.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:963f83df8e69e575b64d67dd447ebbc917db41a14bf38d4593a4183e7aaa8255"}, + {file = "ruff-0.16.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32a5057c7ff3f6e6480a48fccfb3a412a690f48a3d03ac5cf08177d6c2da3ade"}, + {file = "ruff-0.16.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b3dce8d9b0c57c265b91885a66a567d8ea1372e8eb4e250fa8e5e3f579e99cff"}, + {file = "ruff-0.16.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7dc651db49283c69f8e72c834eec4fe5573e4c646856aebece0ce385dceb2a80"}, + {file = "ruff-0.16.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3817b87dbcabc92f13b05019257c5b89b5b4d51b5fb20f56fb5235ceb723cd07"}, + {file = "ruff-0.16.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9fce1499134b2c8c68e5166f95705a5812062bb93aacc5f9873bb1a27084bc7"}, + {file = "ruff-0.16.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2d812e482f5a7e02eee26cd73d2a37ebbdf47d795ea63ba1b89110ae93e9fb3"}, + {file = "ruff-0.16.4-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:6baaf984aa7976edf93d3b627fe2d1d22ee94bbca05fa6f90fc76d73924e3454"}, + {file = "ruff-0.16.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:bdfcf0b28662eb890372d50f92c283bb94e67e7635ed93c7fd533970acff7b2b"}, + {file = "ruff-0.16.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b66b02cb9b04f537643cadf5768e5f98dc461890d530cb67113d71c8c76e605d"}, + {file = "ruff-0.16.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8528bf9a4b291a60bf02ea453511e8ce6215bd2b982ee80405b66b008b6c30a0"}, + {file = "ruff-0.16.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:fbd85d2875fdd67e833213a651f613bbf25303abf6aa822a5121f4531195678d"}, + {file = "ruff-0.16.4-py3-none-win32.whl", hash = "sha256:312769988007aaeb8e189b443ccdd03c0e6374489e053467be6d96518ebff76e"}, + {file = "ruff-0.16.4-py3-none-win_amd64.whl", hash = "sha256:05d9d27a18c4bcbefada602480ec9e01e0bc949d432e0ced5df77edac195919c"}, + {file = "ruff-0.16.4-py3-none-win_arm64.whl", hash = "sha256:a3a61621c9b6f6a89573e938a080e648f1695baa3f58570a3a707bc51ff65a21"}, + {file = "ruff-0.16.4.tar.gz", hash = "sha256:13171aa9d9af2240ee3504e639de73122c67e74036de5ba2e1d01422cd17e3dc"}, ] [[package]] @@ -2946,14 +2946,14 @@ files = [ [[package]] name = "towncrier" -version = "24.8.0" +version = "25.8.0" description = "Building newsfiles for your project." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "towncrier-24.8.0-py3-none-any.whl", hash = "sha256:9343209592b839209cdf28c339ba45792fbfe9775b5f9c177462fd693e127d8d"}, - {file = "towncrier-24.8.0.tar.gz", hash = "sha256:013423ee7eed102b2f393c287d22d95f66f1a3ea10a4baa82d298001a7f18af3"}, + {file = "towncrier-25.8.0-py3-none-any.whl", hash = "sha256:b953d133d98f9aeae9084b56a3563fd2519dfc6ec33f61c9cd2c61ff243fb513"}, + {file = "towncrier-25.8.0.tar.gz", hash = "sha256:eef16d29f831ad57abb3ae32a0565739866219f1ebfbdd297d32894eb9940eb1"}, ] [package.dependencies] @@ -3158,4 +3158,4 @@ optionals = ["jinja2", "jsonschema", "legacycrypt", "napalm", "napalm", "rpds-py [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.15" -content-hash = "15c6e194fae6de97dd0b0f5677d23b4f8f33914f1d502026e36332d4e91161ae" +content-hash = "3cbcfe476ccbba7c200e04803408516f8d925dab47f1821806d4b78ad5ee0131" diff --git a/pyproject.toml b/pyproject.toml index dd5accfa..30165f98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,12 +19,17 @@ classifiers = [ "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", ] + include = [ "LICENSE", "README.md", "netutils/protocols.json" ] +packages = [ + { include = "netutils" }, +] + [tool.poetry.dependencies] python = ">=3.10,<3.15" napalm = [ @@ -44,17 +49,17 @@ optionals = ["jinja2", "jsonschema", "legacycrypt", "napalm", "rpds-py"] [tool.poetry.group.dev.dependencies] coverage = "*" -invoke = "*" pytest = "*" mock = "*" mypy = "*" pyyaml = "^6.0.1" -pylint = "*" -yamllint = "*" +pylint = "~4.0.6" +yamllint = "~1.38.0" +invoke = "~3.0.3" toml = "^0.10.2" attrs = "^23.2.0" -towncrier = ">=23.6.0,<=24.8.0" -ruff = "0.5.5" +towncrier = "~25.8.0" +ruff = "~0.16.2" Markdown = "*" [tool.poetry.group.docs.dependencies] @@ -83,6 +88,9 @@ mkdocstrings-python = "1.13.0" griffe = "1.1.1" mkdocs-python-classy = "0.1.3" +[tool.poetry.scripts] +netutils = 'netutils.cli:main' + [tool.ruff] line-length = 120 target-version = "py310" @@ -181,7 +189,7 @@ show_error_codes = true package = "netutils" directory = "changes" filename = "docs/admin/release_notes/version_1.18.md" -template = "towncrier_template.j2" +template = "development/towncrier_template.j2" start_string = "" issue_format = "[#{issue}](https://github.com/networktocode/netutils/issues/{issue})" diff --git a/tasks.py b/tasks.py index 84805a24..dbe3c315 100644 --- a/tasks.py +++ b/tasks.py @@ -36,10 +36,10 @@ def is_truthy(arg): { "netutils": { "project_name": "netutils", - "python_ver": "3.10", + "python_ver": "3.14", "local": is_truthy(os.getenv("INVOKE_NETUTILS_LOCAL", "false")), "image_name": "netutils", - "image_ver": os.getenv("INVOKE_PARSER_IMAGE_VER", "latest"), + "image_ver": os.getenv("INVOKE_NETUTILS_IMAGE_VER", "latest"), "pwd": Path(__file__).parent, } } @@ -66,13 +66,14 @@ def task_wrapper(function=None): return task_wrapper -def run_command(context, exec_cmd, port=None): +def run_command(context, exec_cmd, port=None, rm=True): """Wrapper to run the invoke task commands. Args: context ([invoke.task]): Invoke task object. exec_cmd ([str]): Command to run. port (int): Used to serve local docs. + rm (bool): Whether to remove the container after running the command. Returns: result (obj): Contains Invoke result from running task. @@ -86,12 +87,12 @@ def run_command(context, exec_cmd, port=None): ) if port: result = context.run( - f"docker run -it -p {port} -v {context.netutils.pwd}:/local {context.netutils.image_name}:{context.netutils.image_ver} sh -c '{exec_cmd}'", + f"docker run -it {'--rm' if rm else ''} -p {port} -v {context.netutils.pwd}:/local {context.netutils.image_name}:{context.netutils.image_ver} sh -c '{exec_cmd}'", pty=True, ) else: result = context.run( - f"docker run -it -v {context.netutils.pwd}:/local {context.netutils.image_name}:{context.netutils.image_ver} sh -c '{exec_cmd}'", + f"docker run -it {'--rm' if rm else ''} -v {context.netutils.pwd}:/local {context.netutils.image_name}:{context.netutils.image_ver} sh -c '{exec_cmd}'", pty=True, ) @@ -188,6 +189,16 @@ def pytest(context, pattern=None, label=None): exec_cmd = " && ".join([doc_test_cmd, pytest_cmd, coverage_cmd]) run_command(context, exec_cmd) + doc_test_cmd = "pytest -vv --doctest-modules netutils/" + pytest_cmd = "coverage run --source=netutils -m pytest" + if pattern: + pytest_cmd += "".join([f" -k {_pattern}" for _pattern in pattern]) + if label: + pytest_cmd += "".join([f" {_label}" for _label in label]) + coverage_cmd = "coverage report" + exec_cmd = " && ".join([doc_test_cmd, pytest_cmd, coverage_cmd]) + run_command(context, exec_cmd) + @task(aliases=("a",)) def autoformat(context): @@ -340,14 +351,22 @@ def docs(context): @task( help={ "version": "Version of netutils to generate the release notes for.", + "date": "Date of the release (default: today).", + "keep": "Keep existing release notes files. Useful for testing. (default: False).", } ) -def generate_release_notes(context, version=""): +def generate_release_notes(context, version="", date="", keep=False): """Generate Release Notes using Towncrier.""" command = "poetry run towncrier build" - if version: - command += f" --version {version}" - else: - command += " --version `poetry version -s`" + if not version: + version = context.run("poetry version --short", hide=True).stdout.strip() + command += f" --version {version}" + if date: + command += f" --date {date}" + command += " --keep" if keep else " --yes" + + version_major_minor = ".".join(version.split(".")[:2]) + context.run(f"poetry run python development/bin/ensure_release_notes.py --version {version_major_minor}") + # Due to issues with git repo ownership in the containers, this must always run locally. context.run(command)