Commit 18361092 authored by David Johnson's avatar David Johnson

In docker image emulabization, attempt to combine COPY instructions.

We now try to emulate any simple COPY <src> <dst> instructions via rsync
prior to image build.

This *does* mean that artifact builder scripts must be careful to create
all necessary dirs according to the base image semantics, because the
base image content is not there when we emulate the COPY instructions.
For instance, many of the modified Dockerfile-runit and
runit-artifacts.sh files depended on built runit packages being
installed into /tmp in the final image -- but they didn't create the
/tmp dir because the COPY instruction they used was running atop a
fully-populated base image that already had /tmp.  Thus, the
runit-artifacts.sh scripts had to be changed to create /tmp with the
proper permissions.
parent 96faef13
......@@ -46,10 +46,13 @@ echo "packager ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
sudo -u packager sh $DIRNAME/alpine3/runit-packager.sh
cd /tmp/runit/x86_64
cp -p *.apk $DESTDIR/
mkdir -p $DESTDIR/tmp
chmod root:root $DESTDIR/tmp
chmod 1777 $DESTDIR/tmp
cp -p *.apk $DESTDIR/tmp
# cd ../..
# cp -p *.deb $DESTDIR/
# cp -p *.deb $DESTDIR/tmp
# rm -rf runit *.deb *.dsc
#
......@@ -64,7 +67,7 @@ sudo -u packager abuild checksum
sudo -u packager -H abuild -r
cp -p /home/packager/packages/tmp/x86_64/shadow-4*.apk \
/home/packager/packages/tmp/x86_64/shadow-uidmap-4*.apk \
$DESTDIR
$DESTDIR/tmp
$DIRNAME/alpine/cleanup.sh
......
COPY artifacts/runit/runit-*.rpm /tmp/
COPY artifacts/runit/ /
......@@ -23,7 +23,10 @@ else
fi
cd runit
./redhat/build.sh
cp -p ~/rpmbuild/RPMS/*/*.rpm $DESTDIR/
mkdir -p $DESTDIR/tmp
chmod root:root $DESTDIR/tmp
chmod 1777 $DESTDIR/tmp
cp -p ~/rpmbuild/RPMS/*/*.rpm $DESTDIR/tmp
cd /tmp
rm -rf runit ~/rpmbuild
......
COPY artifacts/runit/runit*.deb /tmp/
COPY artifacts/runit/ /
......@@ -43,7 +43,10 @@ fi
tar -xzf runit-2.1.2.tar.gz --strip-components=1
dpkg-buildpackage -uc -us
cd ..
cp -p *.deb $DESTDIR/
mkdir -p $DESTDIR/tmp
chmod root:root $DESTDIR/tmp
chmod 1777 $DESTDIR/tmp
cp -p *.deb $DESTDIR/tmp
rm -rf runit *.deb *.dsc
$DIRNAME/debian/cleanup.sh
......
COPY artifacts/runit/runit*.deb /tmp/
COPY artifacts/runit/ /
......@@ -43,7 +43,10 @@ fi
tar -xzf runit-2.1.2.tar.gz --strip-components=1
dpkg-buildpackage -uc -us
cd ..
cp -p *.deb $DESTDIR/
mkdir -p $DESTDIR/tmp
chmod root:root $DESTDIR/tmp
chmod 1777 $DESTDIR/tmp
cp -p *.deb $DESTDIR/tmp
rm -rf runit *.deb *.dsc
$DIRNAME/debian/cleanup.sh
......
COPY artifacts/runit/runit*.deb /tmp/
COPY artifacts/runit/ /
......@@ -43,7 +43,10 @@ fi
tar -xzf runit-2.1.2.tar.gz --strip-components=1
dpkg-buildpackage -uc -us
cd ..
cp -p *.deb $DESTDIR/
mkdir -p $DESTDIR/tmp
chmod root:root $DESTDIR/tmp
chmod 1777 $DESTDIR/tmp
cp -p *.deb $DESTDIR/tmp
rm -rf runit *.deb *.dsc
$DIRNAME/ubuntu/cleanup.sh
......
COPY artifacts/runit/runit*.deb /tmp/
COPY artifacts/runit/ /
......@@ -43,7 +43,10 @@ fi
tar -xzf runit-2.1.2.tar.gz --strip-components=1
dpkg-buildpackage -uc -us
cd ..
cp -p *.deb $DESTDIR/
mkdir -p $DESTDIR/tmp
chmod root:root $DESTDIR/tmp
chmod 1777 $DESTDIR/tmp
cp -p *.deb $DESTDIR/tmp
rm -rf runit *.deb *.dsc
$DIRNAME/ubuntu/cleanup.sh
......
COPY artifacts/runit/runit*.deb /tmp/
COPY artifacts/runit/ /
......@@ -43,7 +43,10 @@ fi
tar -xzf runit-2.1.2.tar.gz --strip-components=1
dpkg-buildpackage -uc -us
cd ..
cp -p *.deb $DESTDIR/
mkdir -p $DESTDIR/tmp
chmod root:root $DESTDIR/tmp
chmod 1777 $DESTDIR/tmp
cp -p *.deb $DESTDIR/tmp
rm -rf runit *.deb *.dsc
$DIRNAME/ubuntu/cleanup.sh
......
......@@ -212,6 +212,11 @@ my $USE_MACVLAN_CNET = 0;
# We try to use $IP instead of $BRCTL.
#
my $USE_BRCTL = 0;
#
# Attempt to replace simple COPY instructions from Dockerfile- fragments
# in image augmentation/emulabization with a single COPY.
#
my $COPY_OPTIMIZE = 1;
##
## Detected configuration variables.
......@@ -5302,7 +5307,7 @@ sub emulabizeImage($;$$$$$$$$$)
#
# First, we are descended FROM the base image.
#
print DFD "FROM $image\n\n";
print DFD "FROM $image\n";
#
# When user is unspecified Docker defaults to root,
......@@ -5311,7 +5316,7 @@ sub emulabizeImage($;$$$$$$$$$)
# However we also must set user back to the Dockerfile's spec
# for entrypoint/cmd ops
#
print DFD "USER root\n\n";
print DFD "USER root\n";
#
# Then, if this is emulabization core or full, add an
......@@ -5320,12 +5325,13 @@ sub emulabizeImage($;$$$$$$$$$)
#
if ($emulabization eq DOCKER_EMULABIZE_CORE()
|| $emulabization eq DOCKER_EMULABIZE_FULL()) {
print DFD "ONBUILD RUN /usr/local/etc/emulab/prepare -M\n\n";
print DFD "ONBUILD RUN /usr/local/etc/emulab/prepare -M\n";
}
#
# Second, copy in all the Dockerfile fragments.
#
my @copies = ();
$cwd = getcwd();
chdir($DOCKERFILES);
foreach my $f (@dfiles) {
......@@ -5333,14 +5339,47 @@ sub emulabizeImage($;$$$$$$$$$)
or fatal("could not open $f to copy into $dockerfile");
my @lines = <FD>;
close(FD);
print DFD join("",@lines)."\n\n";
my @tlines = ();
foreach my $dfline (@lines) {
chomp($dfline);
if ($dfline =~ /^\s*COPY\s+([^\s]+)\s+(.+)$/) {
push(@copies,[$1,$2]);
}
else {
push(@tlines,$dfline);
}
}
if (@tlines > 0) {
print DFD join("\n",@tlines)."\n";
}
}
chdir($cwd);
#
# Next create COPY and RUN commands.
#
print DFD "COPY fs/ /\n";
if ($COPY_OPTIMIZE) {
$cwd = getcwd();
chdir($cdir);
mkdir("combined-fs");
foreach my $sdref (@copies) {
my ($src,$dst) = ($sdref->[0],$sdref->[1]);
if ($dst =~ /^[^\/]/) {
$dst = "combined-fs/$dst";
}
else {
$dst = "combined-fs$dst";
}
mysystem("rsync -a $src $dst");
}
mysystem("rsync -a fs/ combined-fs/");
chdir($cwd);
print DFD "COPY combined-fs/ /\n";
}
else {
print DFD "COPY fs/ /\n";
}
my $runcmd = "";
foreach my $ruc (@runscripts) {
my $dn = dirname($ruc);
......@@ -5371,7 +5410,7 @@ sub emulabizeImage($;$$$$$$$$$)
|| $curzation eq '' || $curzation eq DOCKER_EMULABIZE_NONE()) {
$runcmd .= " && cp -pv /etc/passwd /etc/group /etc/shadow /etc/gshadow /etc/emulab";
}
print DFD "RUN /bin/sh -c '$runcmd'\n\n";
print DFD "RUN /bin/sh -c '$runcmd'\n";
close(DFD);
# We could just send the bytes to the daemon (tar -C $cdir -c . |),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment