[PATCH] s3-mirror: avoid potential unexpected shell-expansion, dedup
by Pavel Raiskup
Doing this has risk that '*' expands to directories:
$> x='echo *'
$> $x
bin boot dev etc home
So it is better to use bash array (we have bash shebang anyways):
$> x=( echo '*' )
$> "${x[@]}"
*
Also de-duplicate few things so it is easier to concentrate on the code.
---
roles/s3-mirror/files/s3.sh | 288 +++++++++++++-----------------------
1 file changed, 106 insertions(+), 182 deletions(-)
diff --git a/roles/s3-mirror/files/s3.sh b/roles/s3-mirror/files/s3.sh
index c70defb52..163e3dc36 100644
--- a/roles/s3-mirror/files/s3.sh
+++ b/roles/s3-mirror/files/s3.sh
@@ -3,207 +3,131 @@
# LGPL
# Author: Rick Elrod <relrod(a)redhat.com>
-# first run this command that syncs, but does not delete.
+base_cmd=(
+ aws s3 sync
+ --no-follow-symlinks
+ --exclude "*/.snapshot/*"
+ --exclude "*/source/*"
+ --exclude "*/SRPMS/*"
+ --exclude "*/debug/*"
+ --exclude "*/beta/*"
+ --exclude "*/ppc/*"
+ --exclude "*/ppc64/*"
+ --exclude "*/repoview/*"
+ --exclude "*/Fedora/*"
+ --exclude "*/EFI/*"
+ --exclude "*/core/*"
+ --exclude "*/extras/*"
+ --exclude "*/LiveOS/*"
+ --exclude "*/development/rawhide/*"
+ --exclude "*/releases/8/*"
+ --exclude "*/releases/9/*"
+ --exclude "*/releases/10/*"
+ --exclude "*/releases/11/*"
+ --exclude "*/releases/12/*"
+ --exclude "*/releases/13/*"
+ --exclude "*/releases/14/*"
+ --exclude "*/releases/15/*"
+ --exclude "*/releases/16/*"
+ --exclude "*/releases/17/*"
+ --exclude "*/releases/18/*"
+ --exclude "*/releases/19/*"
+ --exclude "*/releases/20/*"
+ --exclude "*/releases/21/*"
+ --exclude "*/releases/22/*"
+ --exclude "*/releases/23/*"
+ --exclude "*/releases/24/*"
+ --exclude "*/releases/25/*"
+ --exclude "*/releases/26/*"
+ --exclude "*/releases/27/*"
+ --exclude "*/releases/28/*"
+ --exclude "*/releases/29/*"
+ --exclude "*/updates/8/*"
+ --exclude "*/updates/9/*"
+ --exclude "*/updates/10/*"
+ --exclude "*/updates/11/*"
+ --exclude "*/updates/12/*"
+ --exclude "*/updates/13/*"
+ --exclude "*/updates/14/*"
+ --exclude "*/updates/15/*"
+ --exclude "*/updates/16/*"
+ --exclude "*/updates/17/*"
+ --exclude "*/updates/18/*"
+ --exclude "*/updates/19/*"
+ --exclude "*/updates/20/*"
+ --exclude "*/updates/21/*"
+ --exclude "*/updates/22/*"
+ --exclude "*/updates/23/*"
+ --exclude "*/updates/24/*"
+ --exclude "*/updates/25/*"
+ --exclude "*/updates/26/*"
+ --exclude "*/updates/27/*"
+ --exclude "*/updates/28/*"
+ --exclude "*/updates/29/*"
+ --exclude "*/updates/testing/8/*"
+ --exclude "*/updates/testing/9/*"
+ --exclude "*/updates/testing/10/*"
+ --exclude "*/updates/testing/11/*"
+ --exclude "*/updates/testing/12/*"
+ --exclude "*/updates/testing/13/*"
+ --exclude "*/updates/testing/14/*"
+ --exclude "*/updates/testing/15/*"
+ --exclude "*/updates/testing/16/*"
+ --exclude "*/updates/testing/17/*"
+ --exclude "*/updates/testing/18/*"
+ --exclude "*/updates/testing/19/*"
+ --exclude "*/updates/testing/20/*"
+ --exclude "*/updates/testing/21/*"
+ --exclude "*/updates/testing/22/*"
+ --exclude "*/updates/testing/23/*"
+ --exclude "*/updates/testing/24/*"
+ --exclude "*/updates/testing/25/*"
+ --exclude "*/updates/testing/26/*"
+ --exclude "*/updates/testing/27/*"
+ --exclude "*/updates/testing/28/*"
+ --exclude "*/updates/testing/29/*"
+)
+
+# First run this command that syncs, but does not delete.
# It also excludes repodata.
-CMD1="aws s3 sync \
- --exclude */repodata/* \
- --exclude */.snapshot/* \
- --exclude */source/* \
- --exclude */SRPMS/* \
- --exclude */debug/* \
- --exclude */beta/* \
- --exclude */ppc/* \
- --exclude */ppc64/* \
- --exclude */repoview/* \
- --exclude */Fedora/* \
- --exclude */EFI/* \
- --exclude */core/* \
- --exclude */extras/* \
- --exclude */LiveOS/* \
- --exclude */development/rawhide/* \
- --exclude */releases/8/* \
- --exclude */releases/9/* \
- --exclude */releases/10/* \
- --exclude */releases/11/* \
- --exclude */releases/12/* \
- --exclude */releases/13/* \
- --exclude */releases/14/* \
- --exclude */releases/15/* \
- --exclude */releases/16/* \
- --exclude */releases/17/* \
- --exclude */releases/18/* \
- --exclude */releases/19/* \
- --exclude */releases/20/* \
- --exclude */releases/21/* \
- --exclude */releases/22/* \
- --exclude */releases/23/* \
- --exclude */releases/24/* \
- --exclude */releases/25/* \
- --exclude */releases/26/* \
- --exclude */releases/27/* \
- --exclude */releases/28/* \
- --exclude */releases/29/* \
- --exclude */updates/8/* \
- --exclude */updates/9/* \
- --exclude */updates/10/* \
- --exclude */updates/11/* \
- --exclude */updates/12/* \
- --exclude */updates/13/* \
- --exclude */updates/14/* \
- --exclude */updates/15/* \
- --exclude */updates/16/* \
- --exclude */updates/17/* \
- --exclude */updates/18/* \
- --exclude */updates/19/* \
- --exclude */updates/20/* \
- --exclude */updates/21/* \
- --exclude */updates/22/* \
- --exclude */updates/23/* \
- --exclude */updates/24/* \
- --exclude */updates/25/* \
- --exclude */updates/26/* \
- --exclude */updates/27/* \
- --exclude */updates/28/* \
- --exclude */updates/29/* \
- --exclude */updates/testing/8/* \
- --exclude */updates/testing/9/* \
- --exclude */updates/testing/10/* \
- --exclude */updates/testing/11/* \
- --exclude */updates/testing/12/* \
- --exclude */updates/testing/13/* \
- --exclude */updates/testing/14/* \
- --exclude */updates/testing/15/* \
- --exclude */updates/testing/16/* \
- --exclude */updates/testing/17/* \
- --exclude */updates/testing/18/* \
- --exclude */updates/testing/19/* \
- --exclude */updates/testing/20/* \
- --exclude */updates/testing/21/* \
- --exclude */updates/testing/22/* \
- --exclude */updates/testing/23/* \
- --exclude */updates/testing/24/* \
- --exclude */updates/testing/25/* \
- --exclude */updates/testing/26/* \
- --exclude */updates/testing/27/* \
- --exclude */updates/testing/28/* \
- --exclude */updates/testing/29/* \
- --no-follow-symlinks \
- "
- #--dryrun \
+CMD1=( "${base_cmd[@]}" --exclude "*/repodata/*" )
# Next we run this command which also includes repodata.
-CMD2="aws s3 sync \
- --exclude */.snapshot/* \
- --exclude */source/* \
- --exclude */SRPMS/* \
- --exclude */debug/* \
- --exclude */beta/* \
- --exclude */ppc/* \
- --exclude */ppc64/* \
- --exclude */repoview/* \
- --exclude */Fedora/* \
- --exclude */EFI/* \
- --exclude */core/* \
- --exclude */extras/* \
- --exclude */LiveOS/* \
- --exclude */development/rawhide/* \
- --exclude */releases/8/* \
- --exclude */releases/9/* \
- --exclude */releases/10/* \
- --exclude */releases/11/* \
- --exclude */releases/12/* \
- --exclude */releases/13/* \
- --exclude */releases/14/* \
- --exclude */releases/15/* \
- --exclude */releases/16/* \
- --exclude */releases/17/* \
- --exclude */releases/18/* \
- --exclude */releases/19/* \
- --exclude */releases/20/* \
- --exclude */releases/21/* \
- --exclude */releases/22/* \
- --exclude */releases/23/* \
- --exclude */releases/24/* \
- --exclude */releases/25/* \
- --exclude */releases/26/* \
- --exclude */releases/27/* \
- --exclude */releases/28/* \
- --exclude */releases/29/* \
- --exclude */updates/8/* \
- --exclude */updates/9/* \
- --exclude */updates/10/* \
- --exclude */updates/11/* \
- --exclude */updates/12/* \
- --exclude */updates/13/* \
- --exclude */updates/14/* \
- --exclude */updates/15/* \
- --exclude */updates/16/* \
- --exclude */updates/17/* \
- --exclude */updates/18/* \
- --exclude */updates/19/* \
- --exclude */updates/20/* \
- --exclude */updates/21/* \
- --exclude */updates/22/* \
- --exclude */updates/23/* \
- --exclude */updates/24/* \
- --exclude */updates/25/* \
- --exclude */updates/26/* \
- --exclude */updates/27/* \
- --exclude */updates/28/* \
- --exclude */updates/29/* \
- --exclude */updates/testing/8/* \
- --exclude */updates/testing/9/* \
- --exclude */updates/testing/10/* \
- --exclude */updates/testing/11/* \
- --exclude */updates/testing/12/* \
- --exclude */updates/testing/13/* \
- --exclude */updates/testing/14/* \
- --exclude */updates/testing/15/* \
- --exclude */updates/testing/16/* \
- --exclude */updates/testing/17/* \
- --exclude */updates/testing/18/* \
- --exclude */updates/testing/19/* \
- --exclude */updates/testing/20/* \
- --exclude */updates/testing/21/* \
- --exclude */updates/testing/22/* \
- --exclude */updates/testing/23/* \
- --exclude */updates/testing/24/* \
- --exclude */updates/testing/25/* \
- --exclude */updates/testing/26/* \
- --exclude */updates/testing/27/* \
- --exclude */updates/testing/28/* \
- --exclude */updates/testing/29/* \
- --no-follow-symlinks \
- "
- #--dryrun \
+CMD2=( "${base_cmd[@]}" )
+
+# Then we delete old RPMs and old metadata (but after invalidating caches).
+CMD3=( "${base_cmd[@]}" --delete )
+
+S3_MIRROR=s3-mirror-us-west-1-02.fedoraproject.org
+DIST_ID=E2KJMDC0QAJDMU
# Sync EPEL
-#echo $CMD /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
+#echo $CMD /srv/pub/epel/ s3://$S3_MIRROR/pub/epel/
echo "Starting EPEL sync at $(date)" >> /var/log/s3-mirror/timestamps
-$CMD1 /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
-$CMD2 /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
+"${CMD1[@]}" /srv/pub/epel/ "s3://$S3_MIRROR/pub/epel/"
+"${CMD2[@]}" /srv/pub/epel/ "s3://$S3_MIRROR/pub/epel/"
echo "Ending EPEL sync at $(date)" >> /var/log/s3-mirror/timestamps
for file in $(echo /srv/pub/epel/6/*/repodata/repomd.xml | sed 's#/srv##g'); do
- aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file"
+ aws cloudfront create-invalidation --distribution-id "$DIST_ID" --paths "$file"
done
for file in $(echo /srv/pub/epel/7/*/repodata/repomd.xml | sed 's#/srv##g'); do
- aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file"
+ aws cloudfront create-invalidation --distribution-id "$DIST_ID" --paths "$file"
done
for file in $(echo /srv/pub/epel/8/*/repodata/repomd.xml | sed 's#/srv##g'); do
- aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file"
+ aws cloudfront create-invalidation --distribution-id "$DIST_ID" --paths "$file"
done
-$CMD2 --delete /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
+"${CMD3[@]}" /srv/pub/epel/ "s3://$S3_MIRROR/pub/epel/"
# Sync Fedora
-#echo $CMD /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
+#echo $CMD /srv/pub/fedora/ s3://$S3_MIRROR/pub/fedora/
echo "Starting Fedora sync at $(date)" >> /var/log/s3-mirror/timestamps
-$CMD1 /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
-$CMD2 /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
+"${CMD1[@]}" /srv/pub/fedora/ "s3://$S3_MIRROR/pub/fedora/"
+"${CMD2[@]}" /srv/pub/fedora/ "s3://$S3_MIRROR/pub/fedora/"
echo "Ending Fedora sync at $(date)" >> /var/log/s3-mirror/timestamps
for file in $(echo /srv/pub/fedora/linux/updates/*/*/*/repodata/repomd.xml | sed 's#/srv##g'); do
- aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file"
+ aws cloudfront create-invalidation --distribution-id "$DIST_ID" --paths "$file"
done
-$CMD2 --delete /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
+"${CMD3[@]}" /srv/pub/fedora/ s3://$S3_MIRROR/pub/fedora/
--
2.25.1
4 years, 1 month
MirrorManager upgraded to 0.13
by Adrian Reber
I have upgraded MirrorManager in staging to 0.12, saw that the
webinterface broke, released 0.13, upgraded staging to 0.13 and after
seeing that everything still worked I also upgraded the production
instance.
It contains a couple of small changes which were already active (EPEL 8
modular repository support) in production, but also a security fix which
easily allowed viewing the report_mirror password of all other sites.
The biggest change is that I disabled report_mirror for public mirrors.
Private mirrors still need to run report_mirror, but for public mirrors
the information submitted via report_mirror is ignored. The main reason
for dropping report_mirror for public mirrors is that we regularly have
seen problems of mirrors being disabled by the crawler because they were
out of date and then being re-enabled by report_mirror. The problem was
that report_mirror only submits a list of up to date directories without
the actual content of the directories (to reduce I/O on the mirrors).
Another addition is that MirrorManager can now filter out paths which
should be ignored when detecting a version. MirrorManager regularly
detected new versions in pub/alt like 25757 or 25748 which were IoT test
trees or risc-v trees. So pub/alt is now filtered out when detecting new
versions.
Please let me know if anything breaks.
Adrian
4 years, 1 month
ODCS prod upgrade tomorrow - 2020-03-20 8:00 UTC
by Jan Kaluza
Hi,
I will be upgrading ODCS prod to new version which is already running
on staging. The update will happen tomorrow - 2020-03-20 at 8:00 UTC.
ODCS composes submitted around that time might fail. I presume the
upgrade should take at max one hour.
I will write email here before the upgrade starts and after it ends.
I will also try to find out someone with "root" access to assist me in
case something goes wrong. If you know you will be available during
the Europe's morning tomorrow, please let me know.
Regards,
Jan Kaluza
4 years, 1 month
Fedora 32 Beta Freeze now in effect
by Kevin Fenzi
Greetings.
We are now in the infrastructure freeze leading up to the Fedora 32
Beta release. This is a pre release freeze.
We do this to ensure that our infrastructure is stable and ready to
release the Fedora 32 Beta when it's available.
You can see a list of hosts that do not freeze by checking out the
ansible repo and running the freezelist script:
git clone
https://infrastructure.fedoraproject.org/infra/ansible.git
ansible/scripts/freezelist -i inventory
Any hosts listed as freezes is frozen until 2020-03-17 (or later if
release slips). Frozen hosts should have no changes made to them without
a sign-off on the change from at least 2 sysadmin-main or rel-eng
members, along with (in most cases) a patch of the exact change to be
made to this list.
Thanks,
kevin
4 years, 1 month
[PATCH] s3-mirror: fix missing trailing /
by Kevin Fenzi
This is causing f31 updates to not be synced. The cron job reports:
Subject: Cron <s3-mirror@mm-backend01> /usr/local/bin/lock-wrapper s3sync-updates-current "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/31/Everything/x86_64/os" 2>&1 |
/usr/local/bin/nag-once s3-updates-current.sh 1d 2>&1
Syntax: /usr/local/bin/s3-sync-path.sh /pub/path/to/sync/
NOTE! Path must end with a trailing /
Signed-off-by: Kevin Fenzi <kevin(a)scrye.com>
---
roles/s3-mirror/tasks/main.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/roles/s3-mirror/tasks/main.yml b/roles/s3-mirror/tasks/main.yml
index 5da7a02..395679b 100644
--- a/roles/s3-mirror/tasks/main.yml
+++ b/roles/s3-mirror/tasks/main.yml
@@ -69,7 +69,7 @@
- name: s3sync cron - updates for current
cron: name="s3sync-updates-current" minute="0" hour="3,9,15,21" user="s3-mirror"
- job='/usr/local/bin/lock-wrapper s3sync-updates-current "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/{{ FedoraCycleNumber|int }}/Everything/x86_64/os" 2>&1 | /usr/local/bin/nag-once s3-updates-current.sh 1d 2>&1'
+ job='/usr/local/bin/lock-wrapper s3sync-updates-current "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/{{ FedoraCycleNumber|int }}/Everything/x86_64/os/" 2>&1 | /usr/local/bin/nag-once s3-updates-current.sh 1d 2>&1'
cron_file=s3-updates-current.sh
when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
tags:
--
1.8.3.1
4 years, 1 month