de.dentrassi.build.apt.repo.AptWriter Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of apt-repo Show documentation
Show all versions of apt-repo Show documentation
Create APT based repositories
/*
* Copyright 2014 Jens Reimann.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package de.dentrassi.build.apt.repo;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.TimeZone;
import java.util.zip.GZIPInputStream;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.ar.ArArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.filefilter.AndFileFilter;
import org.apache.commons.io.filefilter.CanReadFileFilter;
import org.apache.commons.io.filefilter.FileFileFilter;
import org.apache.commons.io.filefilter.SuffixFileFilter;
import org.bouncycastle.crypto.Digest;
import org.bouncycastle.crypto.digests.MD5Digest;
import org.bouncycastle.crypto.digests.SHA1Digest;
import org.bouncycastle.crypto.digests.SHA256Digest;
import org.vafer.jdeb.Console;
import org.vafer.jdeb.debian.BinaryPackageControlFile;
/**
* An APT repository writer
*
* This class takes all files from the source directory and converts it to an
* APT repository in another directory. The target directory should be empty or
* not existing since it will overwrite everything with the name state from the
* source directory.
*
*
* Here is what this class can do:
*
* - Copy all source files to a "pool"
* - Extract the metadata and write Packages files
* - Create Release files for components and distributions
* - Create checksum for all files
*
*
*
* At the moment this class is still missing some functionality:
*
* - Signing is not implemented
* - Compression of index files is not implemented
* - And maybe a few other things
*
*
*
* @author Jens Reimann
*/
public class AptWriter
{
private final Configuration configuration;
private File pool;
private File dists;
private interface Digester
{
public Digest create ();
public String getName ();
}
private static class SimpleDigester implements Digester
{
private final String name;
private final Class clazz;
public SimpleDigester ( final String name, final Class clazz )
{
this.name = name;
this.clazz = clazz;
}
@Override
public String getName ()
{
return this.name;
}
@Override
public Digest create ()
{
try
{
return this.clazz.newInstance ();
}
catch ( final Exception e )
{
throw new RuntimeException ( e );
}
}
}
private final List digestersRelease = new LinkedList ();
private final List digestersPackage = new LinkedList ();
private static final DateFormat DF;
private final Map>> files = new HashMap>> ();
private final Console console;
static
{
DF = new SimpleDateFormat ( "EEE, dd MMM YYYY HH:mm:ss z", Locale.US );
DF.setTimeZone ( TimeZone.getTimeZone ( "UTC" ) );
}
public AptWriter ( final Configuration configuration, final Console console )
{
this.console = console;
this.configuration = configuration.clone ();
this.digestersRelease.add ( new SimpleDigester ( "MD5Sum", MD5Digest.class ) );
this.digestersRelease.add ( new SimpleDigester ( "SHA1", SHA1Digest.class ) );
this.digestersRelease.add ( new SimpleDigester ( "SHA256", SHA256Digest.class ) );
this.digestersPackage.add ( new SimpleDigester ( "MD5sum", MD5Digest.class ) ); // yes, this is really the difference
this.digestersPackage.add ( new SimpleDigester ( "SHA1", SHA1Digest.class ) );
this.digestersPackage.add ( new SimpleDigester ( "SHA256", SHA256Digest.class ) );
}
public void build () throws Exception
{
if ( this.configuration.getTargetFolder ().exists () )
{
throw new IllegalStateException ( "The target path must not exists: " + this.configuration.getTargetFolder () );
}
if ( !this.configuration.getSourceFolder ().isDirectory () )
{
throw new IllegalStateException ( "The source path must exists and must be a directory: " + this.configuration.getTargetFolder () );
}
this.configuration.validate ();
this.configuration.getTargetFolder ().mkdirs ();
this.pool = new File ( this.configuration.getTargetFolder (), "pool" );
this.dists = new File ( this.configuration.getTargetFolder (), "dists" );
this.pool.mkdirs ();
this.dists.mkdirs ();
final FileFilter debFilter = new AndFileFilter ( //
Arrays.asList ( //
CanReadFileFilter.CAN_READ, //
FileFileFilter.FILE, //
new SuffixFileFilter ( ".deb" ) //
) //
);
for ( final File packageFile : this.configuration.getSourceFolder ().listFiles ( debFilter ) )
{
processPackageFile ( packageFile );
}
writePackageLists ();
}
private void writePackageLists () throws IOException
{
for ( final Distribution dist : this.configuration.getDistributions () )
{
for ( final Component comp : dist.getComponents () )
{
final Map> fileList = this.files.get ( comp );
for ( final Map.Entry> entry : fileList.entrySet () )
{
writePackageList ( dist, comp, entry.getKey (), entry.getValue () );
}
}
writeRelease ( dist );
}
}
private void writeRelease ( final Distribution dist ) throws IOException
{
final File dir = new File ( this.dists, dist.getName () );
final DistributionReleaseFile rf = new DistributionReleaseFile ();
rf.set ( "Codename", dist.getName () );
rf.set ( "Origin", dist.getOrigin () );
rf.set ( "Label", dist.getLabel () );
rf.set ( "Description", dist.getDescription () );
rf.set ( "Components", join ( dist.getComponents () ) );
rf.set ( "Architectures", join ( this.configuration.getArchitectures () ) );
rf.set ( "Date", DF.format ( new Date () ) );
for ( final Digester d : this.digestersRelease )
{
rf.set ( d.getName (), digestPackageLists ( rf, d, dist ) );
}
final FileOutputStream os = new FileOutputStream ( new File ( dir, "Release" ) );
try
{
os.write ( rf.toString ().getBytes ( "UTF-8" ) );
}
finally
{
os.close ();
}
}
private String digestPackageLists ( final DistributionReleaseFile rf, final Digester d, final Distribution dist ) throws IOException
{
final StringWriter sw = new StringWriter ();
final PrintWriter pw = new PrintWriter ( sw );
final File distDir = new File ( this.dists, dist.getName () ).getCanonicalFile ();
pw.println (); // start with a newline
for ( final Component comp : dist.getComponents () )
{
for ( final String arch : this.configuration.getArchitectures () )
{
File dir = new File ( this.dists, dist.getName () );
dir = new File ( dir, comp.getName () );
dir = new File ( dir, "binary-" + arch );
digestPackageList ( pw, d, distDir, new File ( dir, "Packages" ).getCanonicalFile () );
digestPackageList ( pw, d, distDir, new File ( dir, "Release" ).getCanonicalFile () );
}
}
pw.close ();
return sw.toString ();
}
private void digestPackageList ( final PrintWriter pw, final Digester d, final File distDir, final File file ) throws IOException
{
if ( !file.exists () )
{
return;
}
final String relativeDir = file.getAbsolutePath ().substring ( distDir.getAbsolutePath ().length () + 1 ); // +1 for the leading/trailing slash
final long size = file.length ();
pw.format ( " %s %20s %s", digest ( file, d.create () ), size, relativeDir );
pw.println ();
}
private String join ( final Collection items )
{
if ( items == null )
{
return null;
}
final StringBuilder sb = new StringBuilder ();
boolean first = true;
for ( final Object item : items )
{
if ( first )
{
first = false;
}
else
{
sb.append ( ' ' );
}
sb.append ( item );
}
return sb.toString ();
}
private void writePackageList ( final Distribution distribution, final Component component, final String architecture, final List files ) throws IOException
{
File dir = new File ( this.dists, distribution.getName () );
dir = new File ( dir, component.getName () );
dir = new File ( dir, "binary-" + architecture );
dir.mkdirs ();
// Packages
final File packagesFile = new File ( dir, "Packages" );
this.console.info ( "Writing: " + packagesFile );
final PrintStream ps1 = new PrintStream ( packagesFile );
try
{
for ( final BinaryPackagePackagesFile cf : files )
{
ps1.println ( cf.toString () );
}
}
finally
{
ps1.close ();
}
// Release
final File releaseFile = new File ( dir, "Release" );
this.console.info ( "Writing: " + releaseFile );
final ComponentReleaseFile crf = new ComponentReleaseFile ();
crf.set ( "Component", component.getName () );
crf.set ( "Architecture", architecture );
crf.set ( "Label", component.getLabel () );
crf.set ( "Origin", component.getDistribution ().getOrigin () );
final FileOutputStream os = new FileOutputStream ( releaseFile );
try
{
os.write ( crf.toString ().getBytes ( "UTF-8" ) );
}
finally
{
os.close ();
}
}
protected void processPackageFile ( final File packageFile ) throws Exception
{
final BinaryPackagePackagesFile cf = readArtifact ( packageFile );
final Component component = findComponent ( cf );
if ( component == null )
{
return; // skip
}
this.console.debug ( "Processing: " + cf );
copyArtifact ( component, packageFile, cf );
final String arch = cf.get ( "Architecture" );
if ( "all".equals ( arch ) )
{
for ( final String ae : this.configuration.getArchitectures () )
{
registerPackage ( component, ae, cf );
}
}
else
{
if ( this.configuration.getArchitectures ().contains ( arch ) )
{
registerPackage ( component, arch, cf );
}
}
}
/**
* Get the component that this package is assigned to
*
* Note: This method is called twice at the moment. It must return the same
* result for the same package data.
*
*
* @param cf
* the package file data, may be null
* @return the component or null if the package should be
* ignored
*/
protected Component findComponent ( final BinaryPackagePackagesFile cf )
{
if ( cf == null )
{
return null;
}
// at the moment we allow only one distribution and one component
// you may override this behavior right here
return this.configuration.getDistributions ().iterator ().next ().getComponents ().iterator ().next ();
}
private void registerPackage ( final Component component, final String architecture, final BinaryPackagePackagesFile cf )
{
Map> fileList = this.files.get ( component );
if ( fileList == null )
{
fileList = new HashMap> ();
this.files.put ( component, fileList );
}
List arch = fileList.get ( architecture );
if ( arch == null )
{
arch = new LinkedList ();
fileList.put ( architecture, arch );
}
arch.add ( cf );
}
private BinaryPackagePackagesFile readArtifact ( final File packageFile ) throws Exception
{
final ArArchiveInputStream in = new ArArchiveInputStream ( new FileInputStream ( packageFile ) );
try
{
ArchiveEntry ar;
while ( ( ar = in.getNextEntry () ) != null )
{
if ( !ar.getName ().equals ( "control.tar.gz" ) )
{
continue;
}
final TarArchiveInputStream inputStream = new TarArchiveInputStream ( new GZIPInputStream ( in ) );
try
{
TarArchiveEntry te;
while ( ( te = inputStream.getNextTarEntry () ) != null )
{
if ( !te.getName ().equals ( "./control" ) )
{
continue;
}
return convert ( new BinaryPackageControlFile ( inputStream ), packageFile );
}
}
finally
{
inputStream.close ();
}
}
}
finally
{
IOUtils.closeQuietly ( in );
}
return null;
}
private BinaryPackagePackagesFile convert ( final BinaryPackageControlFile cf, final File packageFile ) throws Exception
{
final BinaryPackagePackagesFile pf = new BinaryPackagePackagesFile ( cf.toString () );
for ( final Digester d : this.digestersPackage )
{
pf.set ( d.getName (), digest ( packageFile, d.create () ) );
}
final Component component = findComponent ( pf );
if ( component == null )
{
return null;
}
final File targetFile = makeTargetFile ( component, packageFile, cf.get ( "Package" ) );
final String filename = targetFile.toString ().substring ( this.configuration.getTargetFolder ().toString ().length () + 1 );
pf.set ( "Filename", filename );
pf.set ( "Size", "" + packageFile.length () );
return pf;
}
public static String digest ( final File file, final Digest digest ) throws IOException
{
InputStream in = null;
try
{
final byte[] buffer = new byte[4096];
in = new FileInputStream ( file );
int rc;
while ( ( rc = in.read ( buffer ) ) > 0 )
{
digest.update ( buffer, 0, rc );
}
final byte[] dv = new byte[digest.getDigestSize ()];
digest.doFinal ( dv, 0 );
final StringBuilder sb = new StringBuilder ();
for ( final byte b : dv )
{
sb.append ( String.format ( "%02x", b ) );
}
return sb.toString ();
}
finally
{
IOUtils.closeQuietly ( in );
}
}
private void copyArtifact ( final Component component, final File packageFile, final BinaryPackagePackagesFile cf ) throws IOException
{
final String name = cf.get ( "Package" );
final File targetFile = makeTargetFile ( component, packageFile, name );
this.console.info ( "Copy artifact: " + targetFile );
targetFile.mkdirs ();
Files.copy ( packageFile.toPath (), targetFile.toPath (), StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING );
}
private File makeTargetFile ( final Component component, final File packageFile, final String packageName )
{
File targetFile = new File ( this.pool, component.getName () );
targetFile = new File ( targetFile, packageName.substring ( 0, 1 ) );
targetFile = new File ( targetFile, packageName );
targetFile = new File ( targetFile, packageFile.getName () );
return targetFile.getAbsoluteFile ();
}
}