salt: update to 3007.1.

Closes: #52887 [via git-merge-pr]
This commit is contained in:
Laurens Vanderhoven 2024-11-02 17:14:26 +01:00 committed by Toyam Cox
parent 2c65490650
commit 216e3ee521
6 changed files with 387 additions and 48 deletions

View file

@ -0,0 +1,53 @@
From 0f69a5a227bfba6ced8a3826d69d556967967fcc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
<marmarek@invisiblethingslab.com>
Date: Wed, 18 Sep 2024 04:54:24 +0200
Subject: [PATCH] Fix Python3.13 compatibility regarding urllib.parse module
Python 3.13 fixed handling relative paths in urllib.parse module.
Specifically, relative file URL is now constructed as file:path instead
of converting it to absolute file:///path. This breaks
salt.utils.url.create which expects file:/// specifically. The mismatch
results in for example changing salt://top.sls into salt://.sls and thus
not finding the top file.
Fix this by handling both prefixes.
Relevant python change: https://github.com/python/cpython/issues/85110
Fixes: #66898
---
changelog/66898.fixed.md | 1 +
salt/utils/url.py | 5 ++---
2 files changed, 3 insertions(+), 3 deletions(-)
create mode 100644 changelog/66898.fixed.md
diff --git a/changelog/66898.fixed.md b/changelog/66898.fixed.md
new file mode 100644
index 000000000000..2549d5e00ed1
--- /dev/null
+++ b/changelog/66898.fixed.md
@@ -0,0 +1 @@
+Fixed Python 3.13 compatibility regarding urllib.parse module
diff --git a/salt/utils/url.py b/salt/utils/url.py
index 478d8e911c2b..839db611c972 100644
--- a/salt/utils/url.py
+++ b/salt/utils/url.py
@@ -4,7 +4,7 @@
import re
import sys
-from urllib.parse import urlparse, urlunparse
+from urllib.parse import urlparse, urlunparse, urlunsplit
import salt.utils.data
import salt.utils.path
@@ -46,8 +46,7 @@ def create(path, saltenv=None):
path = salt.utils.data.decode(path)
query = f"saltenv={saltenv}" if saltenv else ""
- url = salt.utils.data.decode(urlunparse(("file", "", path, "", query, "")))
- return "salt://{}".format(url[len("file:///") :])
+ return f'salt://{salt.utils.data.decode(urlunsplit(("", "", path, query, "")))}'
def is_escaped(url):

View file

@ -0,0 +1,239 @@
From 32bfb196c75c45564982ea0746021fb4dce8c688 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
<marmarek@invisiblethingslab.com>
Date: Thu, 15 Aug 2024 14:57:01 +0200
Subject: [PATCH 1/2] Use timezone-aware datetime objects, for UTC too
datetime.datetime.utcnow() is deprecated in Python 3.12, and it's
recommended to switch to timezone-aware objects, so do this. It also
simplifies local time handling, as .astimezone() method can be used
instead of calculating timezone_delta manually.
Part of #65604
---
salt/grains/core.py | 4 +--
salt/state.py | 34 ++++++++------------
salt/utils/jid.py | 2 +-
salt/utils/pkg/rpm.py | 5 ++-
tests/pytests/unit/grains/test_core.py | 4 ++-
tests/pytests/unit/state/test_state_basic.py | 9 ++++--
tests/unit/utils/test_jid.py | 6 ++++
7 files changed, 37 insertions(+), 27 deletions(-)
diff --git a/salt/grains/core.py b/salt/grains/core.py
index 7afcbd5cbae8..7e0084b42c66 100644
--- a/salt/grains/core.py
+++ b/salt/grains/core.py
@@ -2906,12 +2906,12 @@ def ip_fqdn():
if not ret["ipv" + ipv_num]:
ret[key] = []
else:
- start_time = datetime.datetime.utcnow()
+ start_time = datetime.datetime.now(tz=datetime.timezone.utc)
try:
info = socket.getaddrinfo(_fqdn, None, socket_type)
ret[key] = list({item[4][0] for item in info})
except (OSError, UnicodeError):
- timediff = datetime.datetime.utcnow() - start_time
+ timediff = datetime.datetime.now(tz=datetime.timezone.utc) - start_time
if timediff.seconds > 5 and __opts__["__role"] == "master":
log.warning(
'Unable to find IPv%s record for "%s" causing a %s '
diff --git a/salt/state.py b/salt/state.py
index f8821c498096..3c1dc3faa642 100644
--- a/salt/state.py
+++ b/salt/state.py
@@ -171,11 +171,9 @@ def _calculate_fake_duration():
Generate a NULL duration for when states do not run
but we want the results to be consistent.
"""
- utc_start_time = datetime.datetime.utcnow()
- local_start_time = utc_start_time - (
- datetime.datetime.utcnow() - datetime.datetime.now()
- )
- utc_finish_time = datetime.datetime.utcnow()
+ utc_start_time = datetime.datetime.now(tz=datetime.timezone.utc)
+ local_start_time = utc_start_time.astimezone()
+ utc_finish_time = datetime.datetime.now(tz=datetime.timezone.utc)
start_time = local_start_time.time().isoformat()
delta = utc_finish_time - utc_start_time
# duration in milliseconds.microseconds
@@ -2153,7 +2151,7 @@ def _call_parallel_target(cls, instance, init_kwargs, name, cdata, low):
instance = cls(**init_kwargs)
# we need to re-record start/end duration here because it is impossible to
# correctly calculate further down the chain
- utc_start_time = datetime.datetime.utcnow()
+ utc_start_time = datetime.datetime.now(tz=datetime.timezone.utc)
instance.format_slots(cdata)
tag = _gen_tag(low)
@@ -2173,10 +2171,9 @@ def _call_parallel_target(cls, instance, init_kwargs, name, cdata, low):
"comment": f"An exception occurred in this state: {trb}",
}
- utc_finish_time = datetime.datetime.utcnow()
- timezone_delta = datetime.datetime.utcnow() - datetime.datetime.now()
- local_finish_time = utc_finish_time - timezone_delta
- local_start_time = utc_start_time - timezone_delta
+ utc_finish_time = datetime.datetime.now(tz=datetime.timezone.utc)
+ local_finish_time = utc_finish_time.astimezone()
+ local_start_time = utc_start_time.astimezone()
ret["start_time"] = local_start_time.time().isoformat()
delta = utc_finish_time - utc_start_time
# duration in milliseconds.microseconds
@@ -2206,8 +2203,8 @@ def _call_parallel_target(cls, instance, init_kwargs, name, cdata, low):
*cdata["args"], **cdata["kwargs"]
)
- utc_start_time = datetime.datetime.utcnow()
- utc_finish_time = datetime.datetime.utcnow()
+ utc_start_time = datetime.datetime.now(tz=datetime.timezone.utc)
+ utc_finish_time = datetime.datetime.now(tz=datetime.timezone.utc)
delta = utc_finish_time - utc_start_time
duration = (delta.seconds * 1000000 + delta.microseconds) / 1000.0
retry_ret["duration"] = duration
@@ -2294,10 +2291,8 @@ def call(self, low, chunks=None, running=None, retries=1):
Call a state directly with the low data structure, verify data
before processing.
"""
- utc_start_time = datetime.datetime.utcnow()
- local_start_time = utc_start_time - (
- datetime.datetime.utcnow() - datetime.datetime.now()
- )
+ utc_start_time = datetime.datetime.now(tz=datetime.timezone.utc)
+ local_start_time = utc_start_time.astimezone()
log.info(
"Running state [%s] at time %s",
low["name"].strip() if isinstance(low["name"], str) else low["name"],
@@ -2486,10 +2481,9 @@ def call(self, low, chunks=None, running=None, retries=1):
self.__run_num += 1
format_log(ret)
self.check_refresh(low, ret)
- utc_finish_time = datetime.datetime.utcnow()
- timezone_delta = datetime.datetime.utcnow() - datetime.datetime.now()
- local_finish_time = utc_finish_time - timezone_delta
- local_start_time = utc_start_time - timezone_delta
+ utc_finish_time = datetime.datetime.now(tz=datetime.timezone.utc)
+ local_finish_time = utc_finish_time.astimezone()
+ local_start_time = utc_start_time.astimezone()
ret["start_time"] = local_start_time.time().isoformat()
delta = utc_finish_time - utc_start_time
# duration in milliseconds.microseconds
diff --git a/salt/utils/jid.py b/salt/utils/jid.py
index 69d926469b98..84f5b3c4a323 100644
--- a/salt/utils/jid.py
+++ b/salt/utils/jid.py
@@ -16,7 +16,7 @@ def _utc_now():
"""
Helper method so tests do not have to patch the built-in method.
"""
- return datetime.datetime.utcnow()
+ return datetime.datetime.now(tz=datetime.timezone.utc)
def gen_jid(opts):
diff --git a/salt/utils/pkg/rpm.py b/salt/utils/pkg/rpm.py
index 7574a068e83c..e07f865fef4d 100644
--- a/salt/utils/pkg/rpm.py
+++ b/salt/utils/pkg/rpm.py
@@ -130,7 +130,10 @@ def parse_pkginfo(line, osarch=None):
if install_time not in ("(none)", "0"):
install_date = (
- datetime.datetime.utcfromtimestamp(int(install_time)).isoformat() + "Z"
+ datetime.datetime.fromtimestamp(
+ int(install_time), datetime.timezone.utc
+ ).isoformat()
+ + "Z"
)
install_date_time_t = int(install_time)
else:
diff --git a/tests/pytests/unit/grains/test_core.py b/tests/pytests/unit/grains/test_core.py
index 8b840738ef7e..07735102e506 100644
--- a/tests/pytests/unit/grains/test_core.py
+++ b/tests/pytests/unit/grains/test_core.py
@@ -17,6 +17,7 @@
import tempfile
import textwrap
import uuid
+import warnings
from collections import namedtuple
import pytest
@@ -2151,7 +2152,8 @@ def _check_type(key, value, ip4_empty, ip6_empty):
salt.utils.network, "ip_addrs6", MagicMock(return_value=net_ip6_mock)
), patch.object(
core.socket, "getaddrinfo", side_effect=_getaddrinfo
- ):
+ ), warnings.catch_warnings():
+ warnings.simplefilter("error")
get_fqdn = core.ip_fqdn()
ret_keys = ["fqdn_ip4", "fqdn_ip6", "ipv4", "ipv6"]
for key in ret_keys:
diff --git a/tests/pytests/unit/state/test_state_basic.py b/tests/pytests/unit/state/test_state_basic.py
index c76a8b950ad2..9c5352bc8bb8 100644
--- a/tests/pytests/unit/state/test_state_basic.py
+++ b/tests/pytests/unit/state/test_state_basic.py
@@ -2,6 +2,8 @@
Test functions in state.py that are not a part of a class
"""
+import warnings
+
import pytest
import salt.state
@@ -139,8 +141,11 @@ def test_state_args_id_not_high():
),
]
)
- ret = salt.state.state_args(id_, state, high)
- assert ret == set()
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ salt.utils.jid.gen_jid({})
+ ret = salt.state.state_args(id_, state, high)
+ assert ret == set()
def test_state_args_state_not_high():
diff --git a/tests/unit/utils/test_jid.py b/tests/unit/utils/test_jid.py
index 347e14113280..faa2383639aa 100644
--- a/tests/unit/utils/test_jid.py
+++ b/tests/unit/utils/test_jid.py
@@ -4,6 +4,7 @@
import datetime
import os
+import warnings
import salt.utils.jid
from tests.support.mock import patch
@@ -49,3 +50,8 @@ def test_deprecation_58225(self):
self.assertEqual(
str(no_opts), "gen_jid() missing 1 required positional argument: 'opts'"
)
+
+ def test_deprecation_65604(self):
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ salt.utils.jid.gen_jid({})
From 90454cc14102b1b9d7cfc8cb2c4147b94b83678f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
<marmarek@invisiblethingslab.com>
Date: Wed, 18 Sep 2024 14:58:14 +0200
Subject: [PATCH 2/2] Add changelog entry
---
changelog/65604.fixed.md | 1 +
1 file changed, 1 insertion(+)
create mode 100644 changelog/65604.fixed.md
diff --git a/changelog/65604.fixed.md b/changelog/65604.fixed.md
new file mode 100644
index 000000000000..2e7a9fe75c0c
--- /dev/null
+++ b/changelog/65604.fixed.md
@@ -0,0 +1 @@
+Fixed some instances of deprecated datetime.datetime.utcnow()

View file

@ -0,0 +1,83 @@
diff --git a/salt/utils/pycrypto.py b/salt/utils/pycrypto.py
index e50ac323eb..f13158a112 100644
--- a/salt/utils/pycrypto.py
+++ b/salt/utils/pycrypto.py
@@ -23,13 +23,6 @@ try:
except ImportError:
HAS_RANDOM = False
-try:
- import crypt
-
- HAS_CRYPT = True
-except (ImportError, PermissionError):
- HAS_CRYPT = False
-
try:
import passlib.context
@@ -101,10 +94,6 @@ def secure_password(
raise CommandExecutionError(str(exc))
-if HAS_CRYPT:
- methods = {m.name.lower(): m for m in crypt.methods}
-else:
- methods = {}
known_methods = ["sha512", "sha256", "blowfish", "md5", "crypt"]
@@ -130,26 +119,6 @@ def _gen_hash_passlib(crypt_salt=None, password=None, algorithm=None):
return ctx.hash(**kwargs)
-def _gen_hash_crypt(crypt_salt=None, password=None, algorithm=None):
- """
- Generate /etc/shadow hash using the native crypt module
- """
- if crypt_salt is None:
- # setting crypt_salt to the algorithm makes crypt generate
- # a salt compatible with the specified algorithm.
- crypt_salt = methods[algorithm]
- else:
- if algorithm != "crypt":
- # all non-crypt algorithms are specified as part of the salt
- crypt_salt = f"${methods[algorithm].ident}${crypt_salt}"
-
- try:
- ret = crypt.crypt(password, crypt_salt)
- except OSError:
- ret = None
- return ret
-
-
def gen_hash(crypt_salt=None, password=None, algorithm=None):
"""
Generate /etc/shadow hash
@@ -159,16 +128,12 @@ def gen_hash(crypt_salt=None, password=None, algorithm=None):
if algorithm is None:
# prefer the most secure natively supported method
- algorithm = crypt.methods[0].name.lower() if HAS_CRYPT else known_methods[0]
+ algorithm = known_methods[0]
if algorithm == "crypt" and crypt_salt and len(crypt_salt) != 2:
log.warning("Hash salt is too long for 'crypt' hash.")
- if HAS_CRYPT and algorithm in methods:
- return _gen_hash_crypt(
- crypt_salt=crypt_salt, password=password, algorithm=algorithm
- )
- elif HAS_PASSLIB and algorithm in known_methods:
+ if HAS_PASSLIB and algorithm in known_methods:
return _gen_hash_passlib(
crypt_salt=crypt_salt, password=password, algorithm=algorithm
)
@@ -177,6 +142,6 @@ def gen_hash(crypt_salt=None, password=None, algorithm=None):
"Cannot hash using '{}' hash algorithm. Natively supported "
"algorithms are: {}. If passlib is installed ({}), the supported "
"algorithms are: {}.".format(
- algorithm, list(methods), HAS_PASSLIB, known_methods
+ algorithm, [], HAS_PASSLIB, known_methods
)
)

View file

@ -1,37 +0,0 @@
This is because:
[4baea1a](https://github.com/saltstack/salt/commit/4baea1a97be0389fabe5307d084579134a1f9b7a)
didn't make it in to 3006.3. As per my comment on the commit,
vendored tornado used an obsolete check for
python version. Upstream tornado no longer does.
Fedora carries this patch to fix salt 3006.3 for py 3.12.
This should be obsolete for 3007
--- a/salt/ext/tornado/netutil.py~ 2023-05-05 12:53:34.000000000 -0500
+++ b/salt/ext/tornado/netutil.py 2023-07-24 11:27:02.376824349 -0500
@@ -54,8 +54,8 @@
elif ssl is None:
ssl_match_hostname = SSLCertificateError = None # type: ignore
else:
- import backports.ssl_match_hostname
- ssl_match_hostname = backports.ssl_match_hostname.match_hostname
+ import urllib3.util.ssl_match_hostname
+ ssl_match_hostname = urllib3.util.ssl_match_hostname
SSLCertificateError = backports.ssl_match_hostname.CertificateError # type: ignore
if hasattr(ssl, 'SSLContext'):
--- a/salt/ext/tornado/netutil.py~ 2023-07-24 11:50:02.836988664 -0500
+++ b/salt/ext/tornado/netutil.py 2023-07-24 11:50:52.217539638 -0500
@@ -56,7 +56,7 @@
else:
import urllib3.util.ssl_match_hostname
ssl_match_hostname = urllib3.util.ssl_match_hostname
- SSLCertificateError = backports.ssl_match_hostname.CertificateError # type: ignore
+ SSLCertificateError = urllib3.util.ssl_match_hostname.CertificateError # type: ignore
if hasattr(ssl, 'SSLContext'):
if hasattr(ssl, 'create_default_context'):

View file

@ -1,10 +1,11 @@
diff --git a/requirements/base.txt b/requirements/base.txt
index c19d8804a2b..62244c35152 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -9,4 +9,4 @@ psutil>=5.0.0
packaging>=21.3
looseversion
# We need contextvars for salt-ssh
@@ -15,7 +15,7 @@
# We need contextvars for salt-ssh.
# Even on python versions which ships with contextvars in the standard library!
-contextvars
+contextvars; python_version < "3.7"
setproctitle>=1.2.3
timelib>=0.2.5

View file

@ -1,11 +1,11 @@
# Template file for 'salt'
pkgname=salt
version=3006.8
revision=2
version=3007.1
revision=1
build_style=python3-module
hostmakedepends="python3-setuptools"
depends="python3-Jinja2 python3-M2Crypto python3-MarkupSafe
python3-aiohttp python3-cherrypy python3-cryptography python3-dateutil
depends="dmidecode pciutils python3-Jinja2 python3-M2Crypto python3-MarkupSafe
python3-aiohttp python3-CherryPy python3-cryptography python3-dateutil
python3-distro python3-gnupg python3-importlib_metadata python3-jmespath
python3-looseversion python3-msgpack python3-openssl python3-packaging
python3-passlib python3-psutil python3-pycryptodomex python3-pyzmq
@ -17,7 +17,7 @@ license="Apache-2.0"
homepage="http://saltstack.org/"
changelog="https://docs.saltstack.com/en/latest/topics/releases/${version}.html"
distfiles="${PYPI_SITE}/s/salt/salt-${version}.tar.gz"
checksum=31629905c8d784bdb9786b6a3f77f9a87330bc56d7b68bebc9a19472d9efd866
checksum=b933ac4cb3e4b1118b46dada55c9cc6bdc6f0f94b4c92877aec44b25c6a28c9a
conf_files="
/etc/salt/cloud.providers.d/digitalocean.conf
/etc/salt/cloud.providers.d/vsphere.conf