Gecko [ goons4good.org ]

Name	Size	Permission	Date
__pycache__	[ DIR ]	drwxr-xr-x	2022-10-11 05:09
decorators	[ DIR ]	drwxr-xr-x	2022-10-11 05:09
dockermod	[ DIR ]	drwxr-xr-x	2022-10-11 05:09
openstack	[ DIR ]	drwxr-xr-x	2022-10-11 05:09
pkg	[ DIR ]	drwxr-xr-x	2022-10-11 05:09
validate	[ DIR ]	drwxr-xr-x	2022-10-11 05:09
__init__.py	237 B	-rw-r--r--	2022-05-16 09:16
aggregation.py	5.17 KB	-rw-r--r--	2022-05-16 09:16
args.py	19.86 KB	-rw-r--r--	2022-05-16 09:16
asynchronous.py	4.07 KB	-rw-r--r--	2022-05-16 09:16
atomicfile.py	5.33 KB	-rw-r--r--	2022-05-16 09:16
aws.py	19.38 KB	-rw-r--r--	2022-05-16 09:16
azurearm.py	11.38 KB	-rw-r--r--	2022-05-16 09:16
beacons.py	517 B	-rw-r--r--	2022-05-16 09:16
boto3_elasticsearch.py	3.73 KB	-rw-r--r--	2022-05-16 09:16
boto3mod.py	8.36 KB	-rw-r--r--	2022-05-16 09:16
boto_elb_tag.py	3.02 KB	-rw-r--r--	2022-05-16 09:16
botomod.py	7.98 KB	-rw-r--r--	2022-05-16 09:16
cache.py	9.81 KB	-rw-r--r--	2022-05-16 09:16
cloud.py	117.04 KB	-rw-r--r--	2022-05-16 09:16
color.py	2.72 KB	-rw-r--r--	2022-05-16 09:16
compat.py	1.81 KB	-rw-r--r--	2022-05-16 09:16
configcomparer.py	3.88 KB	-rw-r--r--	2022-05-16 09:16
configparser.py	10.82 KB	-rw-r--r--	2022-05-16 09:16
context.py	6.8 KB	-rw-r--r--	2022-05-16 09:16
crypt.py	4.71 KB	-rw-r--r--	2022-05-16 09:16
ctx.py	1.42 KB	-rw-r--r--	2022-05-16 09:16
data.py	49.28 KB	-rw-r--r--	2022-05-16 09:16
dateutils.py	2.31 KB	-rw-r--r--	2022-05-16 09:16
debug.py	4.19 KB	-rw-r--r--	2022-05-16 09:16
dictdiffer.py	14.16 KB	-rw-r--r--	2022-05-16 09:16
dicttrim.py	3.9 KB	-rw-r--r--	2022-05-16 09:16
dictupdate.py	11.33 KB	-rw-r--r--	2022-05-16 09:16
dns.py	35.17 KB	-rw-r--r--	2022-05-16 09:16
doc.py	2.25 KB	-rw-r--r--	2022-05-16 09:16
entrypoints.py	2.28 KB	-rw-r--r--	2022-05-16 09:16
environment.py	2.2 KB	-rw-r--r--	2022-05-16 09:16
error.py	1.18 KB	-rw-r--r--	2022-05-16 09:16
etcd_util.py	14.22 KB	-rw-r--r--	2022-05-16 09:16
event.py	52.3 KB	-rw-r--r--	2022-05-16 09:16
extend.py	8.87 KB	-rw-r--r--	2022-05-16 09:16
extmods.py	5.67 KB	-rw-r--r--	2022-05-16 09:16
filebuffer.py	3.15 KB	-rw-r--r--	2022-05-16 09:16
files.py	27.6 KB	-rw-r--r--	2022-05-16 09:16
find.py	22.09 KB	-rw-r--r--	2022-05-16 09:16
fsutils.py	3.29 KB	-rw-r--r--	2022-05-16 09:16
functools.py	5.02 KB	-rw-r--r--	2022-05-16 09:16
gitfs.py	124.49 KB	-rw-r--r--	2022-05-16 09:16
github.py	1.52 KB	-rw-r--r--	2022-05-16 09:16
gzip_util.py	2.86 KB	-rw-r--r--	2022-05-16 09:16
hashutils.py	5.91 KB	-rw-r--r--	2022-05-16 09:16
http.py	34.36 KB	-rw-r--r--	2022-05-16 09:16
iam.py	1.22 KB	-rw-r--r--	2022-05-16 09:16
icinga2.py	754 B	-rw-r--r--	2022-05-16 09:16
idem.py	1.22 KB	-rw-r--r--	2022-05-16 09:16
immutabletypes.py	2.46 KB	-rw-r--r--	2022-05-16 09:16
itertools.py	2.36 KB	-rw-r--r--	2022-05-16 09:16
jid.py	3 KB	-rw-r--r--	2022-05-16 09:16
jinja.py	31.54 KB	-rw-r--r--	2022-05-16 09:16
job.py	6.01 KB	-rw-r--r--	2022-05-16 09:16
json.py	3.78 KB	-rw-r--r--	2022-05-16 09:16
kickstart.py	41.04 KB	-rw-r--r--	2022-05-16 09:16
kinds.py	493 B	-rw-r--r--	2022-05-16 09:16
lazy.py	3.06 KB	-rw-r--r--	2022-05-16 09:16
listdiffer.py	10.9 KB	-rw-r--r--	2022-05-16 09:16
locales.py	2.06 KB	-rw-r--r--	2022-05-16 09:16
mac_utils.py	14.01 KB	-rw-r--r--	2022-05-16 09:16
mako.py	3.63 KB	-rw-r--r--	2022-05-16 09:16
master.py	29.57 KB	-rw-r--r--	2022-05-16 09:16
mattermost.py	1.77 KB	-rw-r--r--	2022-05-16 09:16
memcached.py	3.56 KB	-rw-r--r--	2022-05-16 09:16
migrations.py	1.46 KB	-rw-r--r--	2022-05-16 09:16
mine.py	3.68 KB	-rw-r--r--	2022-05-16 09:16
minion.py	4.13 KB	-rw-r--r--	2022-05-16 09:16
minions.py	43.4 KB	-rw-r--r--	2022-05-16 09:16
mount.py	1.15 KB	-rw-r--r--	2022-05-16 09:16
msazure.py	5.36 KB	-rw-r--r--	2022-05-16 09:16
msgpack.py	4.69 KB	-rw-r--r--	2022-05-16 09:16
nacl.py	13.58 KB	-rw-r--r--	2022-05-16 09:16
namecheap.py	4.32 KB	-rw-r--r--	2022-05-16 09:16
napalm.py	23.79 KB	-rw-r--r--	2022-05-16 09:16
nb_popen.py	7.24 KB	-rw-r--r--	2022-05-16 09:16
network.py	71.07 KB	-rw-r--r--	2022-05-16 09:16
nxos.py	12.94 KB	-rw-r--r--	2022-05-16 09:16
nxos_api.py	4 KB	-rw-r--r--	2022-05-16 09:16
odict.py	13.16 KB	-rw-r--r--	2022-05-16 09:16
oset.py	6.41 KB	-rw-r--r--	2022-05-16 09:16
pagerduty.py	3.03 KB	-rw-r--r--	2022-05-16 09:16
parsers.py	125.31 KB	-rw-r--r--	2022-05-16 09:16
path.py	10.78 KB	-rw-r--r--	2022-05-16 09:16
pbm.py	9.81 KB	-rw-r--r--	2022-05-16 09:16
platform.py	5.16 KB	-rw-r--r--	2022-05-16 09:16
powershell.py	4.15 KB	-rw-r--r--	2022-05-16 09:16
preseed.py	2.64 KB	-rw-r--r--	2022-05-16 09:16
process.py	43.34 KB	-rw-r--r--	2022-05-16 09:16
profile.py	3.21 KB	-rw-r--r--	2022-05-16 09:16
proxy.py	331 B	-rw-r--r--	2022-05-16 09:16
psutil_compat.py	3.65 KB	-rw-r--r--	2022-05-16 09:16
pushover.py	4.51 KB	-rw-r--r--	2022-05-16 09:16
pycrypto.py	5.39 KB	-rw-r--r--	2022-05-16 09:16
pydsl.py	13.74 KB	-rw-r--r--	2022-05-16 09:16
pyobjects.py	10.75 KB	-rw-r--r--	2022-05-16 09:16
reactor.py	19.09 KB	-rw-r--r--	2022-05-16 09:16
reclass.py	752 B	-rw-r--r--	2022-05-16 09:16
roster_matcher.py	3.55 KB	-rw-r--r--	2022-05-16 09:16
rsax931.py	8.07 KB	-rw-r--r--	2022-05-16 09:16
s3.py	8.78 KB	-rw-r--r--	2022-05-16 09:16
saltclass.py	14.27 KB	-rw-r--r--	2022-05-16 09:16
sanitizers.py	2.51 KB	-rw-r--r--	2022-05-16 09:16
schedule.py	70.67 KB	-rw-r--r--	2022-05-16 09:16
schema.py	54.26 KB	-rw-r--r--	2022-05-16 09:16
sdb.py	4.04 KB	-rw-r--r--	2022-05-16 09:16
slack.py	3.57 KB	-rw-r--r--	2022-05-16 09:16
smb.py	11.16 KB	-rw-r--r--	2022-05-16 09:16
smtp.py	3.27 KB	-rw-r--r--	2022-05-16 09:16
ssdp.py	14.75 KB	-rw-r--r--	2022-05-16 09:16
ssh.py	769 B	-rw-r--r--	2022-05-16 09:16
state.py	8.43 KB	-rw-r--r--	2022-05-16 09:16
stringio.py	355 B	-rw-r--r--	2022-05-16 09:16
stringutils.py	16.63 KB	-rw-r--r--	2022-05-16 09:16
systemd.py	5.51 KB	-rw-r--r--	2022-05-16 09:16
templates.py	23.92 KB	-rw-r--r--	2022-05-16 09:16
textformat.py	5.03 KB	-rw-r--r--	2022-05-16 09:16
thin.py	31.64 KB	-rw-r--r--	2022-05-16 09:16
timed_subprocess.py	4.06 KB	-rw-r--r--	2022-05-16 09:16
timeout.py	1.53 KB	-rw-r--r--	2022-05-16 09:16
timeutil.py	2.4 KB	-rw-r--r--	2022-05-16 09:16
url.py	4.96 KB	-rw-r--r--	2022-05-16 09:16
user.py	10.92 KB	-rw-r--r--	2022-05-16 09:16
value.py	247 B	-rw-r--r--	2022-05-16 09:16
vault.py	19.23 KB	-rw-r--r--	2022-05-16 09:16
verify.py	26.97 KB	-rw-r--r--	2022-05-16 09:16
versions.py	16.32 KB	-rw-r--r--	2022-05-16 09:16
virt.py	3.24 KB	-rw-r--r--	2022-05-16 09:16
virtualbox.py	22.43 KB	-rw-r--r--	2022-05-16 09:16
vmware.py	129.74 KB	-rw-r--r--	2022-05-16 09:16
vsan.py	17.18 KB	-rw-r--r--	2022-05-16 09:16
vt.py	30.25 KB	-rw-r--r--	2022-05-16 09:16
vt_helper.py	4.4 KB	-rw-r--r--	2022-05-16 09:16
win_chcp.py	3.7 KB	-rw-r--r--	2022-05-16 09:16
win_dacl.py	94.69 KB	-rw-r--r--	2022-05-16 09:16
win_dotnet.py	4.75 KB	-rw-r--r--	2022-05-16 09:16
win_functions.py	12.69 KB	-rw-r--r--	2022-05-16 09:16
win_lgpo_auditpol.py	8.48 KB	-rw-r--r--	2022-05-16 09:16
win_lgpo_netsh.py	17.87 KB	-rw-r--r--	2022-05-16 09:16
win_network.py	12.41 KB	-rw-r--r--	2022-05-16 09:16
win_osinfo.py	2.83 KB	-rw-r--r--	2022-05-16 09:16
win_pdh.py	13.85 KB	-rw-r--r--	2022-05-16 09:16
win_reg.py	30.05 KB	-rw-r--r--	2022-05-16 09:16
win_runas.py	10.55 KB	-rw-r--r--	2022-05-16 09:16
win_service.py	5.2 KB	-rw-r--r--	2022-05-16 09:16
win_system.py	14.47 KB	-rw-r--r--	2022-05-16 09:16
win_update.py	40.36 KB	-rw-r--r--	2022-05-16 09:16
winapi.py	818 B	-rw-r--r--	2022-05-16 09:16
xdg.py	316 B	-rw-r--r--	2022-05-16 09:16
xmlutil.py	13.91 KB	-rw-r--r--	2022-05-16 09:16
yaml.py	348 B	-rw-r--r--	2022-05-16 09:16
yamldumper.py	3.38 KB	-rw-r--r--	2022-05-16 09:16
yamlencoding.py	1.55 KB	-rw-r--r--	2022-05-16 09:16
yamlloader.py	6.21 KB	-rw-r--r--	2022-05-16 09:16
yamlloader_old.py	8.33 KB	-rw-r--r--	2022-05-16 09:16
yast.py	619 B	-rw-r--r--	2022-05-16 09:16
zeromq.py	1.65 KB	-rw-r--r--	2022-05-16 09:16
zfs.py	19.2 KB	-rw-r--r--	2022-05-16 09:16

Rename

"""
Functions used to sync external modules
"""

import logging
import os
import shutil

import salt.fileclient
import salt.utils.files
import salt.utils.hashutils
import salt.utils.path
import salt.utils.url

log = logging.getLogger(__name__)

def _list_emptydirs(rootdir):
    emptydirs = []
    for root, dirs, files in salt.utils.path.os_walk(rootdir):
        if not files and not dirs:
            emptydirs.append(root)
    return emptydirs

def _listdir_recursively(rootdir):
    file_list = []
    for root, dirs, files in salt.utils.path.os_walk(rootdir):
        for filename in files:
            relpath = os.path.relpath(root, rootdir).strip(".")
            file_list.append(os.path.join(relpath, filename))
    return file_list

def sync(opts, form, saltenv=None, extmod_whitelist=None, extmod_blacklist=None):
    """
    Sync custom modules into the extension_modules directory
    """
    if saltenv is None:
        saltenv = ["base"]

if extmod_whitelist is None:
        extmod_whitelist = opts["extmod_whitelist"]
    elif isinstance(extmod_whitelist, str):
        extmod_whitelist = {form: extmod_whitelist.split(",")}
    elif not isinstance(extmod_whitelist, dict):
        log.error(
            "extmod_whitelist must be a string or dictionary: %s", extmod_whitelist
        )

if extmod_blacklist is None:
        extmod_blacklist = opts["extmod_blacklist"]
    elif isinstance(extmod_blacklist, str):
        extmod_blacklist = {form: extmod_blacklist.split(",")}
    elif not isinstance(extmod_blacklist, dict):
        log.error(
            "extmod_blacklist must be a string or dictionary: %s", extmod_blacklist
        )

if isinstance(saltenv, str):
        saltenv = saltenv.split(",")
    ret = []
    remote = set()
    source = salt.utils.url.create("_" + form)
    mod_dir = os.path.join(opts["extension_modules"], "{}".format(form))
    touched = False
    with salt.utils.files.set_umask(0o077):
        try:
            if not os.path.isdir(mod_dir):
                log.info("Creating module dir '%s'", mod_dir)
                try:
                    os.makedirs(mod_dir)
                except OSError:
                    log.error(
                        "Cannot create cache module directory %s. Check permissions.",
                        mod_dir,
                    )
            fileclient = salt.fileclient.get_file_client(opts)
            for sub_env in saltenv:
                log.info("Syncing %s for environment '%s'", form, sub_env)
                cache = []
                log.info("Loading cache from %s, for %s", source, sub_env)
                # Grab only the desired files (.py, .pyx, .so)
                cache.extend(
                    fileclient.cache_dir(
                        source,
                        sub_env,
                        include_empty=False,
                        include_pat=r"E@\.(pyx?|so|zip)$",
                        exclude_pat=None,
                    )
                )
                local_cache_dir = os.path.join(
                    opts["cachedir"], "files", sub_env, "_{}".format(form)
                )
                log.debug("Local cache dir: '%s'", local_cache_dir)
                for fn_ in cache:
                    relpath = os.path.relpath(fn_, local_cache_dir)
                    relname = os.path.splitext(relpath)[0].replace(os.sep, ".")
                    if (
                        extmod_whitelist
                        and form in extmod_whitelist
                        and relname not in extmod_whitelist[form]
                    ):
                        continue
                    if (
                        extmod_blacklist
                        and form in extmod_blacklist
                        and relname in extmod_blacklist[form]
                    ):
                        continue
                    remote.add(relpath)
                    dest = os.path.join(mod_dir, relpath)
                    log.info("Copying '%s' to '%s'", fn_, dest)
                    if os.path.isfile(dest):
                        # The file is present, if the sum differs replace it
                        hash_type = opts.get("hash_type", "md5")
                        src_digest = salt.utils.hashutils.get_hash(fn_, hash_type)
                        dst_digest = salt.utils.hashutils.get_hash(dest, hash_type)
                        if src_digest != dst_digest:
                            # The downloaded file differs, replace!
                            shutil.copyfile(fn_, dest)
                            ret.append("{}.{}".format(form, relname))
                    else:
                        dest_dir = os.path.dirname(dest)
                        if not os.path.isdir(dest_dir):
                            os.makedirs(dest_dir)
                        shutil.copyfile(fn_, dest)
                        ret.append("{}.{}".format(form, relname))

touched = bool(ret)
            if opts["clean_dynamic_modules"] is True:
                current = set(_listdir_recursively(mod_dir))
                for fn_ in current - remote:
                    full = os.path.join(mod_dir, fn_)
                    if os.path.isfile(full):
                        touched = True
                        os.remove(full)
                # Cleanup empty dirs
                while True:
                    emptydirs = _list_emptydirs(mod_dir)
                    if not emptydirs:
                        break
                    for emptydir in emptydirs:
                        touched = True
                        shutil.rmtree(emptydir, ignore_errors=True)
        except Exception as exc:  # pylint: disable=broad-except
            log.error("Failed to sync %s module: %s", form, exc)
    return ret, touched