Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
emulab
emulab-devel
Commits
ba3ab203
Commit
ba3ab203
authored
Apr 29, 2015
by
Mike Hibler
Browse files
Scripts for crunching over large collections of images.
Work in progress.
parent
a4f6e098
Changes
6
Hide whitespace changes
Inline
Side-by-side
clientside/os/imagezip/TESTS/checksha.pl
0 → 100644
View file @
ba3ab203
#
# Check SHA1 hash file against image.
#
my
$hashall
=
0
;
if
(
@ARGV
<
1
)
{
print
STDERR
"
Usage: $0 directory-of-images
\n
";
exit
(
1
);
}
my
$imagedir
=
$ARGV
[
0
];
if
(
!
-
d
$imagedir
)
{
print
STDERR
"
$imagedir
: not a directory
\n
";
exit
(
1
);
}
my
@files
=
`
cd
$imagedir
; /bin/ls -1 *.ndz*
`;
chomp
@files
;
#print "Found: ", join(' ', @files), "\n";
my
@images
=
();
foreach
my
$file
(
@files
)
{
# straight up ndz
if
(
$file
=~
/\.ndz$/
)
{
push
@images
,
$file
;
next
;
}
# versioned ndz
if
(
$file
=~
/\.ndz:\d+$/
)
{
push
@images
,
$file
;
next
;
}
}
print
STDERR
"
Found
",
int
(
@images
),
"
images
\n
";
foreach
my
$file
(
@images
)
{
print
"
$file
:
";
if
(
!
-
e
"
$imagedir
/
$file
.sha1
")
{
print
"
[FAIL] no signature!
\n
";
next
;
}
# make sure it is the right format:
# SHA1 (fname) = 9f2a0f8160f70a7b29b0e1de2088a38d0f2bc229
my
$sha1
=
`
cat
$imagedir
/
$file
.sha1
`;
chomp
(
$sha1
);
if
(
$sha1
=~
/^SHA1 .* = ([0-9a-f]{40})$/
)
{
$sha1
=
$
1
;
}
else
{
print
"
[FAIL] bogus .sha1 file
\n
";
next
;
}
my
$nsha1
=
`
sha1
$imagedir
/
$file
`;
chomp
(
$nsha1
);
if
(
$nsha1
=~
/^SHA1 .* = ([0-9a-f]{40})$/
)
{
$nsha1
=
$
1
;
}
else
{
print
"
[FAIL] did not correctly compute sha1!
\n
";
next
;
}
if
(
$sha1
eq
$nsha1
)
{
print
"
[OK]
\n
";
}
else
{
print
"
[BAD]
\n
";
}
}
exit
(
0
);
clientside/os/imagezip/TESTS/deltafy.pl
0 → 100644
View file @
ba3ab203
#
# Parse a directory of images looking for those that follow our versioning
# convention. For those, start with the lowest numbered version and create
# deltas for all but the first and last, e.g.:
#
# imagedelta UBUNTU14-64-STD.ndz UBUNTU14-64-STD.ndz:1 UBUNTU14-64-STD.ddz:1
# rm UBUNTU14-64-STD.ndz:1
# ...
# imagedelta UBUNTU14-64-STD.ndz:8 UBUNTU14-64-STD.ndz:9 UBUNTU14-64-STD.ddz:9
# rm UBUNTU14-64-STD.ndz:9
#
my
$frombase
=
1
;
my
$removefull
=
0
;
my
$IMAGEDELTA
=
"
/tmp/imagedelta
";
if
(
@ARGV
<
1
)
{
print
STDERR
"
Usage: $0 directory-of-images
\n
";
exit
(
1
);
}
my
$imagedir
=
$ARGV
[
0
];
if
(
!
-
d
$imagedir
)
{
print
STDERR
"
$imagedir
: not a directory
\n
";
exit
(
1
);
}
my
$tstamp
=
"
deltafy.
"
.
time
();
if
(
!
open
(
ST
,
"
>
$imagedir
/
$tstamp
"))
{
print
STDERR
"
$imagedir
: cannot write to directory
\n
";
exit
(
1
);
}
my
@files
=
`
cd
$imagedir
; /bin/ls -1 *.ndz*
`;
chomp
@files
;
if
(
@files
==
0
)
{
print
STDERR
"
$imagedir
: no images found
\n
";
unlink
(
$tstamp
);
exit
(
1
);
}
my
%filehash
=
map
{
("
$_
"
=>
1
)
}
@files
;
my
%images
=
();
#print STDERR "Files left #1: ", join(' ', keys %filehash), "\n\n";
# Find all the base files
foreach
my
$file
(
@files
)
{
if
(
$file
=~
/^(.*)\.ndz$/
)
{
my
$ibase
=
$
1
;
if
(
-
l
"
$imagedir
/
$file
")
{
print
STDERR
"
$imagedir
: ignoring symlink '
$file
'
\n
";
delete
$filehash
{
$file
};
delete
$filehash
{"
$file
.sig
"};
delete
$filehash
{"
$file
.sha1
"};
next
;
}
$images
{
$ibase
}{'
name
'}
=
$file
;
$images
{
$ibase
}{'
lastvers
'}
=
0
;
@
{
$images
{
$ibase
}{'
versions
'}}
=
();
delete
$filehash
{
$file
};
}
}
#print STDERR "Files left #2: ", join(' ', keys %filehash), "\n\n";
# Find all the versions
foreach
my
$file
(
@files
)
{
next
if
(
!
exists
(
$filehash
{
$file
}));
if
(
$file
=~
/^(.*).ndz:(\d+)$/
)
{
my
(
$ibase
,
$vers
)
=
(
$
1
,
$
2
);
if
(
exists
(
$images
{
$ibase
}))
{
push
@
{
$images
{
$ibase
}{'
versions
'}},
$vers
;
if
(
$vers
>
$images
{
$ibase
}{'
lastvers
'})
{
$images
{
$ibase
}{'
lastvers
'}
=
$vers
;
}
delete
$filehash
{
$file
};
next
;
}
print
STDERR
"
*** version with no base '
$file
', ignoring
\n
";
delete
$filehash
{
$file
};
delete
$filehash
{"
$file
.sig
"};
delete
$filehash
{"
$file
.sha1
"};
next
;
}
}
#print STDERR "Files left #3: ", join(' ', keys %filehash), "\n\n";
# Make sure all versions and signatures exist
foreach
my
$ibase
(
keys
%images
)
{
my
$nukeit
=
0
;
my
%versions
=
map
{
(
$_
=>
1
)
}
@
{
$images
{
$ibase
}{'
versions
'}};
foreach
my
$vers
(
0
..
$images
{
$ibase
}{'
lastvers
'})
{
my
$fbase
=
"
$ibase
.ndz
";
my
$vstr
=
"";
if
(
$vers
>
0
)
{
$vstr
=
"
:
$vers
";
if
(
!
exists
(
$versions
{
$vers
}))
{
if
(
$frombase
)
{
print
STDERR
"
WARNING:
";
}
else
{
print
STDERR
"
***
";
}
print
STDERR
"
no version
$vers
of '
$ibase
' ('
$fbase$vstr
')
";
if
(
!
$frombase
)
{
print
STDERR
"
, ignoring
\n
";
$nukeit
=
1
;
}
else
{
print
STDERR
"
\n
";
delete
$filehash
{"
$fbase$vstr
.sig
"};
delete
$filehash
{"
$fbase$vstr
.sha1
"};
next
;
}
}
else
{
delete
$filehash
{"
$fbase$vstr
"};
}
}
# got sig?
if
(
!
exists
(
$filehash
{"
$fbase$vstr
.sig
"}))
{
if
(
!
$frombase
)
{
print
STDERR
"
*** no signature for
$fbase$vstr
, ignoring
\n
";
$nukeit
=
1
;
}
else
{
print
STDERR
"
WARNING: no signature for
$fbase$vstr
,
"
.
"
ignoring version
\n
";
delete
$filehash
{"
$fbase$vstr
.sha1
"};
next
;
}
}
else
{
delete
$filehash
{"
$fbase$vstr
.sig
"};
}
# what about the hash?
delete
$filehash
{"
$fbase$vstr
.sha1
"};
}
if
(
$nukeit
)
{
delete
$images
{
$ibase
};
}
}
# warn about unknown files
if
(
scalar
(
keys
%filehash
)
>
0
)
{
print
STDERR
"
WARNING: unknown files:
\n
";
}
foreach
my
$file
(
sort
keys
%filehash
)
{
print
STDERR
"
$file
\n
";
}
# what do we have left
foreach
my
$ibase
(
sort
keys
%images
)
{
my
$lvers
=
$images
{
$ibase
}{'
lastvers
'};
print
STDERR
"
$ibase
: image and
$lvers
versions
\n
";
foreach
my
$vers
(
1
..
$images
{
$ibase
}{'
lastvers
'})
{
my
$base
=
"
$ibase
.ndz
";
if
(
$vers
>
1
&&
!
$frombase
)
{
$base
=
"
$ibase
.ndz:
"
.
(
$vers
-
1
);
}
my
$this
=
"
$ibase
.ndz:
$vers
";
if
(
$frombase
&&
(
!
-
e
"
$imagedir
/
$this
"
||
!
-
e
"
$imagedir
/
$this
.sig
"))
{
print
STDERR
"
$ibase
: version
$vers
skipped because of missing files
\n
";
next
;
}
my
$delta
=
"
$ibase
.ddz:
$vers
";
if
(
system
("
$IMAGEDELTA
-SF
$imagedir
/
$base
$imagedir
/
$this
$imagedir
/
$delta
\n
"))
{
print
STDERR
"
*** '
$IMAGEDELTA
-SF
$imagedir
/
$base
$imagedir
/
$this
$imagedir
/
$delta
' failed!
\n
";
}
}
}
exit
(
0
);
clientside/os/imagezip/TESTS/hashem.pl
0 → 100644
View file @
ba3ab203
#
# Create signature files where missing. Check others.
#
my
$hashall
=
0
;
my
$IMAGEHASH
=
"
/tmp/imagehash
";
if
(
@ARGV
<
1
)
{
print
STDERR
"
Usage: $0 directory-of-images
\n
";
exit
(
1
);
}
my
$imagedir
=
$ARGV
[
0
];
if
(
!
-
d
$imagedir
)
{
print
STDERR
"
$imagedir
: not a directory
\n
";
exit
(
1
);
}
my
$tstamp
=
"
hashem.
"
.
time
();
if
(
!
open
(
ST
,
"
>
$imagedir
/
$tstamp
"))
{
print
STDERR
"
$imagedir
: cannot write to directory
\n
";
exit
(
1
);
}
my
$clogfile
=
"
$imagedir
/check.log
";
my
$glogfile
=
"
$imagedir
/generate.log
";
system
("
echo '' >
$clogfile
; date >>
$clogfile
");
system
("
echo '' >
$glogfile
; date >>
$glogfile
");
my
@files
=
`
cd
$imagedir
; /bin/ls -1 *.ndz*
`;
chomp
@files
;
#print "Found: ", join(' ', @files), "\n";
my
@images
=
();
foreach
my
$file
(
@files
)
{
# no symlinks
next
if
(
-
l
"
$imagedir
/
$file
");
# no sha1s
next
if
(
$file
=~
/\.sha1$/
);
# straight up ndz
if
(
$file
=~
/\.ndz$/
)
{
push
@images
,
$file
;
next
;
}
# versioned ndz
if
(
$file
=~
/\.ndz:\d+$/
)
{
push
@images
,
$file
;
next
;
}
}
print
STDERR
"
Found
",
int
(
@images
),
"
images
\n
";
foreach
my
$file
(
@images
)
{
print
"
$file
:
";
if
(
-
e
"
$imagedir
/
$file
.sig
")
{
print
"
found sig...
";
if
(
checksig
(
$file
))
{
print
"
[OK]
\n
";
next
;
}
print
"
[BAD]...re
";
}
print
"
generating...
";
if
(
gensig
(
$file
))
{
print
"
[OK]
\n
";
next
;
}
print
"
[FAIL]
\n
";
}
exit
(
0
);
sub
checksig
($)
{
my
$file
=
shift
;
if
(
system
("
(echo
$file
;
$IMAGEHASH
-SX
$imagedir
/
$file
) >>
$clogfile
2>&1
"))
{
return
0
;
}
return
1
;
}
sub
gensig
($)
{
my
$file
=
shift
;
if
(
system
("
(echo
$file
;
$IMAGEHASH
-cX
$imagedir
/
$file
) >>
$glogfile
2>&1
"))
{
return
0
;
}
return
1
;
}
clientside/os/imagezip/TESTS/testdelta.pl
0 → 100644
View file @
ba3ab203
#
# Given two full images I1 and I2:
#
# I. Basic:
#
# 1. create a shiny new delta image (imagedelta) I2.ddz.v2
# 2. create a shiny new full image (imageundelta) I2.ndz.v2 from I1 and delta
# 3. compare sigs of I2.ndz and I2.ndz.v2
#
# What we learn: imagedelta/undelta don't crash on the image in question,
# delta/undelta are correct at the level of the signature file.
#
# II. More advanced:
#
# 4. lay down I1.ndz on a ramdisk and optionally verify against sig
# 5. lay down I2.ndz on a ramdisk and optionally verify against sig
# 6. create an old school delta image (imagezip -D) I2.ddz from I2 disk, I1 sig
# 7. compare sigs of I2.ddz I2.ddz.v2
# 7a. if sigs don't compare [*], we compate I2.ddz.v2 against the I2 disk
#
# What we learn: imagezip and imagedelta delta creation are consistent.
#
# III. Over the top:
#
# 8. apply delta I2.ddz.v2 to I1 disk
# 9. compare contents of I1 and I2 disks [2]
# 10. load full I2.ndz.v2 image on I3 disk
# 11. compare contents of I2 and I3 disks [3]
#
# What we learn: new format files can be loaded (imagezip), actual on-disk
# contents are correct.
#
# Footnotes:
#
# [1] Since delta signatures generated by imagezip are imperfect,
# the sigs may not match. To ensure their equivalence, we assume
# the imagezip sig is correct and verify the other by checking it
# against the on-disk image.
#
# [2] We cannot just "cmp" the two disks due to inexactness of the
# delta created by imagedelta. It may include unused blocks that
# will get propogate back to disk. Hence, we create new image files
# from both disks using imagezip--with "-F 0" to ensure no free sects
# are included--and byte compare those image files.
#
# [3] As with imagedelta, imageundelta is inexact and may have included
# garbage sectors from the original image. We perform the same
# comparison as in [2] instead.
#
my
$TMPDIR
=
"
/local/tmp
";
my
$LOGDIR
=
"
/local/logs
";
my
@NEEDBINS
=
("
imagezip
",
"
imageunzip
",
"
imagehash
",
"
imagedump
",
"
imagedelta
",
"
imageundelta
");
my
$MAXPHASE
=
2
;
my
$MAXSECTORS
=
(
20
*
1024
*
1024
*
2
);
my
$LVMSTRIPE
=
6
;
my
$checksig
=
0
;
my
$cleanonfail
=
0
;
my
$os
=
`
uname
`;
chomp
(
$os
);
if
(
$os
!~
/^(Linux|FreeBSD)$/
)
{
die
"
Unknown OS '
$os
'
\n
";
}
my
$arch
=
`
uname -m
`;
chomp
(
$arch
);
if
(
$arch
!~
/^(x86_32|x86_64|aarch64|i386|amd64)$/
)
{
die
"
Unknown arch '
$arch
'
\n
";
}
my
$bindir
=
"
/images/bin/
${os}
_
${arch}
";
foreach
my
$bin
(
@NEEDBINS
)
{
if
(
!
-
x
"
$bindir
/
$bin
")
{
die
"
Cannot find
$bindir
/
$bin
\n
";
}
}
if
(
@ARGV
!=
2
)
{
print
STDERR
"
Usage: testdelta.pl imageV1.ndz imageV2.ndz
\n
";
exit
(
1
);
}
my
$uid
=
$>
;
if
(
$uid
==
0
)
{
print
STDERR
"
Do not run as root, script will sudo when necessary
\n
";
exit
(
1
);
}
my
$tstamp
=
time
();
print
"
Logs will be
$LOGDIR
/
$tstamp
.*.log ...
\n
";
my
%image1
=
(
"
name
"
=>
$ARGV
[
0
]
);
my
%image2
=
(
"
name
"
=>
$ARGV
[
1
]
);
my
$rv
=
0
;
$rv
=
phase1
(
\
%image1
,
\
%image2
)
if
(
$MAXPHASE
>
0
);
$rv
=
phase2
(
\
%image1
,
\
%image2
)
if
(
$rv
==
0
&&
$MAXPHASE
>
1
);
$rv
=
phase3
(
\
%image1
,
\
%image2
)
if
(
$rv
==
0
&&
$MAXPHASE
>
2
);
if
(
$rv
==
0
)
{
unmakedisk
(
$image1
{'
name
'},
$image1
{'
disk
'});
unmakedisk
(
$image2
{'
name
'},
$image2
{'
disk
'});
}
exit
(
$rv
);
#
# I. Basic:
#
# 1. create a shiny new delta image (imagedelta) I2.ddz.v2
# 2. create a shiny new full image (imageundelta) I2.ndz.v2 from I1 and delta
# 3. compare sigs of I2.ndz and I2.ndz.v2
#
sub
phase1
($$)
{
my
(
$image1
,
$image2
)
=
@_
;
my
$fixedsig
=
0
;
my
$imagev1
=
$image1
->
{'
name
'};
my
$imagev2
=
$image2
->
{'
name
'};
logit
("
START signature check of both images
");
if
(
sigcheck
(
$imagev1
)
||
sigcheck
(
$imagev2
))
{
logit
("
FAILED signature check of both images
");
return
1
;
}
logit
("
END signature check of both images
");
my
$delta
=
$imagev2
;
$delta
=~
s/ndz/ddz/
;
if
(
$imagev2
eq
$delta
)
{
logit
("
$imagev2
: malformed name, no .ndz
");
return
1
;
}
logit
("
START image2 delta creation
");
again:
if
(
mysystem
("
$bindir
/imagedelta -SVF
$imagev1
$imagev2
$delta
.new
",
0
))
{
logit
("
WARNING: could not generate delta image
");
if
(
!
$fixedsig
)
{
logit
("
recreating signature for
$imagev2
...
");
mysystem
("
mv
$imagev2
.sig
$imagev2
.bak.sig
");
if
(
!
mysystem
("
$bindir
/imagehash -qcX
$imagev2
"))
{
logit
("
validating new signature for
$imagev2
...
");
if
(
!
comparesigfiles
("
$imagev2
.bak
",
$imagev2
,
0
))
{
unlink
("
$imagev2
.bak.sig
");
logit
("
RESTART image2 delta creation
");
$fixedsig
=
1
;
goto
again
;
}
unlink
("
$imagev2
.sig
");
}
mysystem
("
mv
$imagev2
.bak.sig
$imagev2
.sig
");
}
logit
("
FAILED image2 delta creation
");
return
1
;
}
logit
("
END image2 delta creation
");
logit
("
START undelta to image2.new
");
if
(
mysystem
("
$bindir
/imageundelta -SV
$imagev1
$delta
.new
$imagev2
.new
"))
{
logit
("
FAILED undelta to image2.new
");
return
1
;
}
logit
("
END undelta to image2.new
");
logit
("
START signature compare of image2 and image2.new
");
if
(
comparesigfiles
(
$imagev2
,
"
$imagev2
.new
",
1
))
{
logit
("
$imagev2
.new: signature mismatch with
$imagev2
");
logit
("
FAILED signature compare of image2 and image2.new
");
return
1
;
}
logit
("
END signature compare of image2 and image2.new
");
return
0
;
}
#
# II. More advanced:
#
# 4. lay down I1.ndz on a ramdisk and optionally verify against sig
# 5. lay down I2.ndz on a ramdisk and optionally verify against sig
# 6. create an old school delta image (imagezip -D) I2.ddz from I2 disk, I1 sig
# 7. compare sigs of I2.ddz I2.ddz.v2
#
sub
phase2
($$)
{
my
(
$image1
,
$image2
)
=
@_
;
my
$imagev1
=
$image1
->
{'
name
'};
my
$imagev2
=
$image2
->
{'
name
'};
my
$delta
=
$imagev2
;
$delta
=~
s/ndz/ddz/
;
if
(
$imagev2
eq
$delta
)
{
logit
("
$imagev2
: malformed name, no .ndz
");
return
1
;
}
logit
("
START unzip of image1
");
my
(
$devv1
,
$ssizev1
)
=
unzipimage
(
$imagev1
);
if
(
!
$devv1
)
{
logit
("
FAILED unzip of image1
");
return
1
;
}
logit
("
END unzip of image1
");
logit
("
START signature check of image1 on disk
");
if
(
$checksig
)
{
if
(
mysystem
("
$bindir
/imagehash -qX
$imagev1
$devv1
"))
{
logit
("
FAILED signature check of image1 on disk
");
unmakedisk
(
$imagev1
,
$devv1
)
if
(
$cleanonfail
);
return
1
;
}
}
logit
("
END signature check of image1 on disk
");
logit
("
START unzip of image2
");
my
(
$devv2
,
$ssizev2
)
=
unzipimage
(
$imagev2
);
if
(
!
$devv2
)
{
logit
("
FAILED unzip of image2
");
unmakedisk
(
$imagev1
,
$devv1
)
if
(
$cleanonfail
);
return
1
;
}
logit
("
END unzip of image2
");
logit
("
START signature check of image2 on disk
");
if
(
$checksig
)
{
if
(
mysystem
("
$bindir
/imagehash -qX
$imagev2
$devv2
"))
{
logit
("
FAILED signature check of image2 on disk
");
unmakedisk
(
$imagev2
,
$devv2
)
if
(
$cleanonfail
);
unmakedisk
(
$imagev1
,
$devv1
)
if
(
$cleanonfail
);
return
1
;
}
}
logit
("
END signature check of image2 on disk
");
logit
("
START create delta of image2 from disk with imagezip
");
if
(
zipimage
("
$delta
.old
",
"
$imagev1
.sig
",
$devv2
,
$ssizev2
))
{
logit
("
FAILED create delta of image2 from disk with imagezip
");
unmakedisk
(
$imagev2
,
$devv2
)
if
(
$cleanonfail
);
unmakedisk
(
$imagev1
,
$devv1
)
if
(
$cleanonfail
);
return
1
;
}
logit
("
END create delta of image2 from disk with imagezip
");
#
# Hashmaps for delta images produced by imagezip are less than
# perfect as they can split a contiguous range in the new image if
# the corresponding range in the old image was not fully populated
# (and probably under other conditions too!)
#
if
(
-
e
"
$delta
.new
")
{
logit
("
START image sigfile compare for deltas
");
# returns -1 if files are missing
my
$rv
=
comparesigfiles
("
$delta
.old
",
"
$delta
.new
",
0
);
if
(
$rv
>
0
)
{
<