client/tools/spacewalk-remote-utils/spacewalk-remote-utils.spec | 8 java/code/src/com/redhat/rhn/taskomatic/task/repomd/ChannelRepodataWorker.java | 19 java/code/src/com/redhat/rhn/taskomatic/task/repomd/DebRepositoryWriter.java | 91 + java/code/src/com/redhat/rhn/taskomatic/task/repomd/RepositoryWriter.java | 540 ---------- java/code/src/com/redhat/rhn/taskomatic/task/repomd/RpmRepositoryWriter.java | 472 ++++++++ 5 files changed, 630 insertions(+), 500 deletions(-)
New commits: commit 8c177fa5d22e80744fab303d5ef594ad8c03d01b Author: Simon Lukasik slukasik@redhat.com Date: Sat Oct 2 11:55:41 2010 +0200
Remove all from cache, not only rpm specific metadata.
diff --git a/java/code/src/com/redhat/rhn/taskomatic/task/repomd/RepositoryWriter.java b/java/code/src/com/redhat/rhn/taskomatic/task/repomd/RepositoryWriter.java index b03d419..f01d5f2 100644 --- a/java/code/src/com/redhat/rhn/taskomatic/task/repomd/RepositoryWriter.java +++ b/java/code/src/com/redhat/rhn/taskomatic/task/repomd/RepositoryWriter.java @@ -64,7 +64,7 @@ public abstract class RepositoryWriter { public abstract String getCompsFilePath(Channel channel);
/** - * Deletes repomd files + * Deletes repository cache files * @param channelLabelToProcess channel label * @param deleteDir directory to delete */ @@ -72,28 +72,17 @@ public abstract class RepositoryWriter { log.info("Removing " + channelLabelToProcess); String prefix = mountPoint + File.separator + pathPrefix + File.separator + channelLabelToProcess; - File primary = new File(prefix + File.separator + "primary.xml.gz"); - File filelists = new File(prefix + File.separator + "filelists.xml.gz"); - File other = new File(prefix + File.separator + "other.xml.gz"); - File repomd = new File(prefix + File.separator + "repomd.xml"); - File updateinfo = new File(prefix + File.separator + "updateinfo.xml.gz"); - File norepo = new File(prefix + File.separator + "noyumrepo.txt"); File theDirectory = new File(prefix);
- if (!primary.delete()) { - log.info("Couldn't remove " + primary.getAbsolutePath()); - } - if (!filelists.delete()) { - log.info("Couldn't remove " + filelists.getAbsolutePath()); - } - if (!other.delete()) { - log.info("Couldn't remove " + other.getAbsolutePath()); - } - if (!repomd.delete()) { - log.info("Couldn't remove " + repomd.getAbsolutePath()); + String[] children = theDirectory.list(); + if (theDirectory.isDirectory() && children != null) { + for (int i = 0; i < children.length; i ++) { + File file = new File(prefix + File.separator + children[i]); + if (!file.delete()) { + log.info("Couldn't remove " + file.getAbsolutePath()); + } + } } - updateinfo.delete(); - norepo.delete(); if (deleteDir) { if (!theDirectory.delete()) { log.info("Couldn't remove " + prefix);
commit ad9d30c1588c434c9ee41140e11a138e6543213c Author: Simon Lukasik slukasik@redhat.com Date: Fri Oct 1 16:19:40 2010 +0200
Implement isChannelRepodataStale for debian channels.
diff --git a/java/code/src/com/redhat/rhn/taskomatic/task/repomd/DebRepositoryWriter.java b/java/code/src/com/redhat/rhn/taskomatic/task/repomd/DebRepositoryWriter.java index d3480bf..a37a04a 100644 --- a/java/code/src/com/redhat/rhn/taskomatic/task/repomd/DebRepositoryWriter.java +++ b/java/code/src/com/redhat/rhn/taskomatic/task/repomd/DebRepositoryWriter.java @@ -15,8 +15,10 @@ package com.redhat.rhn.taskomatic.task.repomd;
import java.io.File; +import java.util.Date;
import com.redhat.rhn.common.hibernate.HibernateFactory; +import com.redhat.rhn.common.localization.LocalizationService; import com.redhat.rhn.domain.channel.Channel; import com.redhat.rhn.frontend.dto.PackageDto; import com.redhat.rhn.manager.rhnpackage.PackageManager; @@ -44,7 +46,17 @@ public class DebRepositoryWriter extends RepositoryWriter { * @return repodata sanity */ public boolean isChannelRepodataStale(Channel channel) { - return true; + File theFile = new File(mountPoint + File.separator + pathPrefix + + File.separator + channel.getLabel() + File.separator + + "Packages.gz"); + Date fileModifiedDate = new Date(theFile.lastModified()); + // the file Modified date should be getting set when the file + // is moved into the correct location. + log.info("File Modified Date:" + LocalizationService.getInstance(). + formatCustomDate(fileModifiedDate)); + log.info("Channel Modified Date:" + LocalizationService.getInstance(). + formatCustomDate(channel.getLastModified())); + return !fileModifiedDate.equals(channel.getLastModified()); }
/**
commit 2e68c64438e375f40b711346259bc8533918956d Author: Simon Lukasik slukasik@redhat.com Date: Fri Oct 1 13:34:24 2010 +0200
Rpm and Deb metadata creation differs, moving to separate classes.
diff --git a/java/code/src/com/redhat/rhn/taskomatic/task/repomd/ChannelRepodataWorker.java b/java/code/src/com/redhat/rhn/taskomatic/task/repomd/ChannelRepodataWorker.java index 5c021b5..c5162ea 100644 --- a/java/code/src/com/redhat/rhn/taskomatic/task/repomd/ChannelRepodataWorker.java +++ b/java/code/src/com/redhat/rhn/taskomatic/task/repomd/ChannelRepodataWorker.java @@ -27,6 +27,7 @@ import com.redhat.rhn.taskomatic.task.threaded.QueueWorker; import com.redhat.rhn.taskomatic.task.threaded.TaskQueue;
import org.apache.log4j.Logger; +import org.jfree.util.Log;
import java.util.HashMap; import java.util.List; @@ -58,10 +59,20 @@ public class ChannelRepodataWorker implements QueueWorker { String mountPoint = Config.get().getString(ConfigDefaults.REPOMD_CACHE_MOUNT_POINT, "/pub"); channelLabelToProcess = (String) workItem.get("channel_label"); - repoWriter = new RepositoryWriter(prefixPath, mountPoint); - logger.info("Creating ChannelRepodataWorker with prefixPath(" + prefixPath + - "), mountPoint(" + mountPoint + ")" + " for channel_label (" + - channelLabelToProcess + ")"); + + // We need to find out whether to use Rpm or Debian repository + Channel channelToProcess = ChannelFactory.lookupByLabel(channelLabelToProcess); + // if the channelExists in the db still + if (channelToProcess != null && channelToProcess.getChannelArch() + .getArchType().getLabel().equalsIgnoreCase("deb")) { + repoWriter = new DebRepositoryWriter(prefixPath, mountPoint); + } + else { + repoWriter = new RpmRepositoryWriter(prefixPath, mountPoint); + } + logger.info("Creating ChannelRepodataWorker with prefixPath(" + prefixPath + + "), mountPoint(" + mountPoint + ")" + " for channel_label (" + + channelLabelToProcess + ")"); }
/** diff --git a/java/code/src/com/redhat/rhn/taskomatic/task/repomd/DebRepositoryWriter.java b/java/code/src/com/redhat/rhn/taskomatic/task/repomd/DebRepositoryWriter.java new file mode 100644 index 0000000..d3480bf --- /dev/null +++ b/java/code/src/com/redhat/rhn/taskomatic/task/repomd/DebRepositoryWriter.java @@ -0,0 +1,79 @@ +/** + * Copyright (c) 2009--2010 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.rhn.taskomatic.task.repomd; + +import java.io.File; + +import com.redhat.rhn.common.hibernate.HibernateFactory; +import com.redhat.rhn.domain.channel.Channel; +import com.redhat.rhn.frontend.dto.PackageDto; +import com.redhat.rhn.manager.rhnpackage.PackageManager; +import com.redhat.rhn.manager.task.TaskManager; + +/** +* +* @version $Rev $ +* +*/ +public class DebRepositoryWriter extends RepositoryWriter { + + /** + * Constructor takes in pathprefix and mountpoint + * @param pathPrefixIn prefix to package path + * @param mountPointIn mount point package resides + */ + public DebRepositoryWriter(String pathPrefixIn, String mountPointIn) { + super(pathPrefixIn, mountPointIn); + } + + /** + * + * @param channel channel info + * @return repodata sanity + */ + public boolean isChannelRepodataStale(Channel channel) { + return true; + } + + /** + * Create repository for APT + * @param channel + */ + public void writeRepomdFiles(Channel channel) { + PackageManager.createRepoEntrys(channel.getId()); + + String prefix = mountPoint + File.separator + pathPrefix + + File.separator + channel.getLabel() + File.separator; + + // we closed the session, so we need to reload the object + channel = (Channel) HibernateFactory.getSession().get(channel.getClass(), + channel.getId()); + if (!new File(prefix).mkdirs() && !new File(prefix).exists()) { + throw new RepomdRuntimeException("Unable to create directory: " + + prefix); + } + + log.info("Generating new DEB repository for channel " + channel.getLabel()); + DebPackageWriter writer = new DebPackageWriter(channel, prefix); + for (PackageDto pkgDto : TaskManager.getChannelPackageDtos(channel)) { + writer.addPackage(pkgDto); + } + writer.generatePackagesGz(); + } + + public String getCompsFilePath(Channel channel) { + return null; + } +} diff --git a/java/code/src/com/redhat/rhn/taskomatic/task/repomd/RepositoryWriter.java b/java/code/src/com/redhat/rhn/taskomatic/task/repomd/RepositoryWriter.java index 12fcf03..b03d419 100644 --- a/java/code/src/com/redhat/rhn/taskomatic/task/repomd/RepositoryWriter.java +++ b/java/code/src/com/redhat/rhn/taskomatic/task/repomd/RepositoryWriter.java @@ -14,53 +14,22 @@ */ package com.redhat.rhn.taskomatic.task.repomd;
-import com.redhat.rhn.common.conf.Config; -import com.redhat.rhn.common.conf.ConfigDefaults; -import com.redhat.rhn.common.hibernate.HibernateFactory; -import com.redhat.rhn.common.localization.LocalizationService; -import com.redhat.rhn.common.util.StringUtil; import com.redhat.rhn.domain.channel.Channel; -import com.redhat.rhn.domain.channel.ChannelFactory; -import com.redhat.rhn.frontend.dto.PackageDto; -import com.redhat.rhn.manager.channel.ChannelManager; -import com.redhat.rhn.manager.rhnpackage.PackageManager; -import com.redhat.rhn.manager.task.TaskManager;
import org.apache.log4j.Logger;
-import java.io.BufferedWriter; import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.FileWriter; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.security.DigestInputStream; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Date; -import java.util.HashMap; -import java.util.Map;
/** * * @version $Rev $ * */ -public class RepositoryWriter { +public abstract class RepositoryWriter {
- private static final String PRIMARY_FILE = "primary.xml.gz.new"; - private static final String FILELISTS_FILE = "filelists.xml.gz.new"; - private static final String OTHER_FILE = "other.xml.gz.new"; - private static final String REPOMD_FILE = "repomd.xml.new"; - private static final String UPDATEINFO_FILE = "updateinfo.xml.gz.new"; - private static final String NOREPO_FILE = "noyumrepo.txt"; - - private Logger log = Logger.getLogger(RepositoryWriter.class); - private String pathPrefix; - private String mountPoint; - private String checksumtype; + protected Logger log = Logger.getLogger(RepositoryWriter.class); + protected String pathPrefix; + protected String mountPoint;
/** * Constructor takes in pathprefix and mountpoint @@ -73,464 +42,62 @@ public class RepositoryWriter { }
/** - * - * @param channel channel info - * @return repodata sanity - */ - public boolean isChannelRepodataStale(Channel channel) { - File theFile = new File(mountPoint + File.separator + pathPrefix + - File.separator + channel.getLabel() + File.separator + - "repomd.xml"); - Date fileModifiedDate = new Date(theFile.lastModified()); - // the file Modified date should be getting set when the file - // is moved into the correct location. - log.info("File Modified Date:" + LocalizationService.getInstance(). - formatCustomDate(fileModifiedDate)); - log.info("Channel Modified Date:" + LocalizationService.getInstance(). - formatCustomDate(channel.getLastModified())); - return !fileModifiedDate.equals(channel.getLastModified()); - } - /** - * - * @param channel channelinfo for repomd file creation - */ - public void writeRepomdFiles(Channel channel) { - - PackageManager.createRepoEntrys(channel.getId()); - - String prefix = mountPoint + File.separator + pathPrefix + - File.separator + channel.getLabel() + File.separator; - - // we closed the session, so we need to reload the object - channel = (Channel) HibernateFactory.getSession().get(channel.getClass(), - channel.getId()); - if (!new File(prefix).mkdirs() && !new File(prefix).exists()) { - throw new RepomdRuntimeException("Unable to create directory: " + - prefix); - } - if (channel.getChannelArch().getArchType().getLabel().equalsIgnoreCase("deb")) { - log.info("Generating new DEB repository for channel " + channel.getLabel()); - generateDebRepository(channel, prefix); - } - else { - // Get compatible checksumType - this.checksumtype = channel.getChecksumTypeLabel(); - if (checksumtype == null) { - generateBadRepo(channel, prefix); - return; - } - new File(prefix + NOREPO_FILE).delete(); - log.info("Checksum Type Value: " + this.checksumtype); - - // java.security.MessageDigest recognizes: - // MD2, MD5, SHA-1, SHA-256, SHA-384, SHA-512 - String checksumAlgo = this.checksumtype; - if (checksumAlgo.toUpperCase().startsWith("SHA")) { - checksumAlgo = this.checksumtype.substring(0, 3) + "-" + - this.checksumtype.substring(3); - } - // translate sha1 to sha for xml repo files - String checksumLabel = this.checksumtype; - if (checksumLabel.equals("sha1")) { - checksumLabel = "sha"; - } - - log.info("Generating new repository metatada for channel '" + - channel.getLabel() + "' " + channel.getPackageCount() + - " packages, " + channel.getErrataCount() + " updates"); - - CompressingDigestOutputWriter primaryFile; - CompressingDigestOutputWriter filelistsFile; - CompressingDigestOutputWriter otherFile; - - try { - primaryFile = new CompressingDigestOutputWriter( - new FileOutputStream(prefix + PRIMARY_FILE), - checksumAlgo); - filelistsFile = new CompressingDigestOutputWriter( - new FileOutputStream(prefix + FILELISTS_FILE), - checksumAlgo); - otherFile = new CompressingDigestOutputWriter( - new FileOutputStream(prefix + OTHER_FILE), checksumAlgo); - } - catch (IOException e) { - throw new RepomdRuntimeException(e); - } - catch (NoSuchAlgorithmException e) { - throw new RepomdRuntimeException(e); - } - - BufferedWriter primaryBufferedWriter = new BufferedWriter( - new OutputStreamWriter(primaryFile)); - BufferedWriter filelistsBufferedWriter = new BufferedWriter( - new OutputStreamWriter(filelistsFile)); - BufferedWriter otherBufferedWriter = new BufferedWriter( - new OutputStreamWriter(otherFile)); - PrimaryXmlWriter primary = new PrimaryXmlWriter( - primaryBufferedWriter); - FilelistsXmlWriter filelists = new FilelistsXmlWriter( - filelistsBufferedWriter); - OtherXmlWriter other = new OtherXmlWriter(otherBufferedWriter); - Date start = new Date(); - - primary.begin(channel); - filelists.begin(channel); - other.begin(channel); - - for (PackageDto pkgDto : TaskManager.getChannelPackageDtos(channel)) { - primary.addPackage(pkgDto); - filelists.addPackage(pkgDto); - other.addPackage(pkgDto); - try { - primaryFile.flush(); - filelistsFile.flush(); - otherFile.flush(); - } - catch (IOException e) { - throw new RepomdRuntimeException(e); - } - } - primary.end(); - filelists.end(); - other.end(); - try { - primaryBufferedWriter.close(); - filelistsBufferedWriter.close(); - otherBufferedWriter.close(); - } - catch (IOException e) { - throw new RepomdRuntimeException(e); - } - - RepomdIndexData primaryData = new RepomdIndexData(primaryFile - .getCompressedChecksum(), primaryFile - .getUncompressedChecksum(), channel.getLastModified()); - RepomdIndexData filelistsData = new RepomdIndexData(filelistsFile - .getCompressedChecksum(), filelistsFile - .getUncompressedChecksum(), channel.getLastModified()); - RepomdIndexData otherData = new RepomdIndexData(otherFile - .getCompressedChecksum(), otherFile - .getUncompressedChecksum(), channel.getLastModified()); - - log.info("Starting updateinfo generation for '" + channel.getLabel() + '"'); - log.info("Checksum Type Value for generate updateinfo " + this.checksumtype); - RepomdIndexData updateinfoData = generateUpdateinfo(channel, - prefix, checksumAlgo); - - RepomdIndexData groupsData = loadCompsFile(channel, checksumAlgo); - - // Set the type so yum can read and perform checksum - primaryData.setType(checksumLabel); - filelistsData.setType(checksumLabel); - otherData.setType(checksumLabel); - if (updateinfoData != null) { - updateinfoData.setType(checksumLabel); - } - - if (groupsData != null) { - groupsData.setType(checksumLabel); - } - - log.info("Primary xml's type: " + primaryData.getType()); - log.info("filelists xml's type: " + filelistsData.getType()); - log.info("other xml's type: " + otherData.getType()); - - FileWriter indexFile; - - try { - indexFile = new FileWriter(prefix + REPOMD_FILE); - } - catch (IOException e) { - throw new RepomdRuntimeException(e); - } - - RepomdIndexWriter index = new RepomdIndexWriter(indexFile, primaryData, - filelistsData, otherData, updateinfoData, groupsData); - - index.writeRepomdIndex(); - - try { - indexFile.close(); - } - catch (IOException e) { - throw new RepomdRuntimeException(e); - } - - renameFiles(prefix, channel.getLastModified().getTime(), - updateinfoData != null); - - log.info("Repository metadata generation for '" + - channel.getLabel() + "' finished in " + - (int) (new Date().getTime() - start.getTime()) / 1000 + " seconds"); - } - } - - /** - * Deletes existing repo and generates file stating that no repo was generated - * @param channel the channel to do this for - * @param prefix the directory prefix - */ - private void generateBadRepo(Channel channel, String prefix) { - log.warn("No repo will be generated for channel " + channel.getLabel()); - deleteRepomdFiles(channel.getLabel(), false); - try { - FileWriter norepo = new FileWriter(prefix + NOREPO_FILE); - norepo.write("No repo will be generated for channel " + - channel.getLabel() + ".\n"); - norepo.close(); - } - catch (IOException e) { - log.warn("Cannot create " + NOREPO_FILE + " file."); - } - return; - } - - /** - * Create repository for APT - * @param channel - */ - private void generateDebRepository(Channel channel, String prefix) { - DebPackageWriter writer = new DebPackageWriter(channel, prefix); - for (PackageDto pkgDto : TaskManager.getChannelPackageDtos(channel)) { - writer.addPackage(pkgDto); - } - writer.generatePackagesGz(); - } - - private String getCompsRelativeFilename(Channel channel) { - String relativeFilename = null; - - if (channel.getComps() == null) { - relativeFilename = getCompsFilePath(channel); - } - else { - relativeFilename = channel.getComps().getRelativeFilename(); - } - - // if we didn't find anything, let's check channel's original - if (relativeFilename == null && channel.isCloned()) { - // use a hack not to use ClonedChannel and it's getOriginal() method - Long originalChannelId = ChannelManager.lookupOriginalId(channel); - Channel originalChannel = ChannelFactory.lookupById(originalChannelId); - return getCompsRelativeFilename(originalChannel); - } - - return relativeFilename; - } - - /** - * - * @param channel channel indo - * @param checksumAlgo checksum algorithm - * @return repomd index for given channel - */ - private RepomdIndexData loadCompsFile(Channel channel, String checksumAlgo) { - String compsMount = Config.get().getString(ConfigDefaults.MOUNT_POINT); - String relativeFilename = getCompsRelativeFilename(channel); - - if (relativeFilename == null) { - return null; - } - - File compsFile = new File(compsMount + File.separator + relativeFilename); - FileInputStream stream; - try { - stream = new FileInputStream(compsFile); - } - catch (FileNotFoundException e) { - return null; - } - - DigestInputStream digestStream; - try { - digestStream = new DigestInputStream(stream, MessageDigest - .getInstance(checksumAlgo)); - } - catch (NoSuchAlgorithmException nsae) { - throw new RepomdRuntimeException(nsae); - } - byte[] bytes = new byte[10]; - - try { - while (digestStream.read(bytes) != -1) { - // no-op - } - } - catch (IOException e) { - return null; - } - - Date timeStamp = new Date(compsFile.lastModified()); - - return new RepomdIndexData(StringUtil.getHexString(digestStream - .getMessageDigest().digest()), null, timeStamp); - } - - /** - * TODO: This static comps paths should go away once - * we can get the paths directly from hosted through - * satellite-sync and only limit to supporting cloned. - * @param channel channel object - * @return compsPath comps file path + * + * @param channel channel info + * @return repodata sanity */ - public String getCompsFilePath(Channel channel) { - String compsPath = null; - - Map<String, String> compsMapping = new HashMap<String, String>(); - String rootClientPath = "/rhn/kickstart/ks-rhel-x86_64-client-5"; - String rootServerPath = "/rhn/kickstart/ks-rhel-x86_64-server-5"; - compsMapping.put("rhel-x86_64-client-5", - rootClientPath + "/Client/repodata/comps-rhel5-client-core.xml"); - compsMapping.put("rhel-x86_64-client-vt-5", - rootClientPath + "/VT/repodata/comps-rhel5-vt.xml"); - compsMapping.put("rhel-x86_64-client-workstation-5", - rootClientPath + "/Workstation/repodata/comps-rhel5-client-workstation.xml"); - compsMapping.put("rhel-x86_64-server-5", - rootServerPath + "/Server/repodata/comps-rhel5-server-core.xml"); - compsMapping.put("rhel-x86_64-server-vt-5", - rootServerPath + "/VT/repodata/comps-rhel5-vt.xml"); - compsMapping.put("rhel-x86_64-server-cluster-5", - rootServerPath + "/Cluster/repodata/comps-rhel5-cluster.xml"); - compsMapping.put("rhel-x86_64-server-cluster-storage-5", - rootServerPath + "/ClusterStorage/repodata/comps-rhel5-cluster-st.xml"); - - String[] arches = {"i386", "ia64", "s390x", "ppc"}; - Map<String, String> newCompsmap = new HashMap<String, String>(); - for (String k : compsMapping.keySet()) { - for (String arch : arches) { - newCompsmap.put(k.replace("x86_64", arch), - compsMapping.get(k).replace("x86_64", arch)); - } - } - compsMapping.putAll(newCompsmap); - - if (compsMapping.containsKey(channel.getLabel())) { - compsPath = compsMapping.get(channel.getLabel()); - } - - return compsPath; - } + public abstract boolean isChannelRepodataStale(Channel channel);
/** - * Generates update info for given channel - * @param channel channel info - * @param prefix repodata file prefix - * @param checksumtype checksum type - * @return repodata index - */ - private RepomdIndexData generateUpdateinfo(Channel channel, String prefix, - String checksumtypeIn) { - - if (channel.getErrataCount() == 0) { - return null; - } - - CompressingDigestOutputWriter updateinfoFile; - try { - updateinfoFile = new CompressingDigestOutputWriter( - new FileOutputStream(prefix + UPDATEINFO_FILE), checksumtypeIn); - } - catch (FileNotFoundException e) { - throw new RepomdRuntimeException(e); - } - catch (NoSuchAlgorithmException e) { - throw new RepomdRuntimeException(e); - } - catch (IOException e) { - throw new RepomdRuntimeException(e); - } - BufferedWriter updateinfoBufferedWriter = new BufferedWriter( - new OutputStreamWriter(updateinfoFile)); - UpdateInfoWriter updateinfo = new UpdateInfoWriter( - updateinfoBufferedWriter); - updateinfo.getUpdateInfo(channel); - try { - updateinfoBufferedWriter.close(); - } - catch (IOException e) { - throw new RepomdRuntimeException(e); - } - - RepomdIndexData updateinfoData = new RepomdIndexData(updateinfoFile - .getCompressedChecksum(), updateinfoFile - .getUncompressedChecksum(), channel.getLastModified()); - return updateinfoData; - } - - /** - * Renames the repo cache files - * @param prefix path prefix - * @param lastModified file last_modified - * @param doUpdateinfo - */ - private void renameFiles(String prefix, Long lastModified, - Boolean doUpdateinfo) { - File primary = new File(prefix + PRIMARY_FILE); - File filelists = new File(prefix + FILELISTS_FILE); - File other = new File(prefix + OTHER_FILE); - File repomd = new File(prefix + REPOMD_FILE); - - File updateinfo = null; - if (doUpdateinfo) { - updateinfo = new File(prefix + UPDATEINFO_FILE); - } - - if (doUpdateinfo) { - updateinfo.setLastModified(lastModified); - } - - primary.setLastModified(lastModified); - filelists.setLastModified(lastModified); - other.setLastModified(lastModified); - repomd.setLastModified(lastModified); - - if (doUpdateinfo) { - updateinfo.renameTo(new File(prefix + "updateinfo.xml.gz")); - } - - primary.renameTo(new File(prefix + "primary.xml.gz")); - filelists.renameTo(new File(prefix + "filelists.xml.gz")); - other.renameTo(new File(prefix + "other.xml.gz")); - repomd.renameTo(new File(prefix + "repomd.xml")); - - } - - /** - * Deletes repomd files - * @param channelLabelToProcess channel label - * @param deleteDir directory to delete - */ - public void deleteRepomdFiles(String channelLabelToProcess, boolean deleteDir) { - log.info("Removing " + channelLabelToProcess); - String prefix = mountPoint + File.separator + pathPrefix + File.separator + - channelLabelToProcess; - File primary = new File(prefix + File.separator + "primary.xml.gz"); - File filelists = new File(prefix + File.separator + "filelists.xml.gz"); - File other = new File(prefix + File.separator + "other.xml.gz"); - File repomd = new File(prefix + File.separator + "repomd.xml"); - File updateinfo = new File(prefix + File.separator + "updateinfo.xml.gz"); - File norepo = new File(prefix + File.separator + NOREPO_FILE); - File theDirectory = new File(prefix); - - if (!primary.delete()) { - log.info("Couldn't remove " + primary.getAbsolutePath()); - } - if (!filelists.delete()) { - log.info("Couldn't remove " + filelists.getAbsolutePath()); - } - if (!other.delete()) { - log.info("Couldn't remove " + other.getAbsolutePath()); - } - if (!repomd.delete()) { - log.info("Couldn't remove " + repomd.getAbsolutePath()); - } - updateinfo.delete(); - norepo.delete(); - if (deleteDir) { - if (!theDirectory.delete()) { - log.info("Couldn't remove " + prefix); - } - } - } - + * + * @param channel channelinfo for repomd file creation + */ + public abstract void writeRepomdFiles(Channel channel); + + /** + * TODO: This static comps paths should go away once + * we can get the paths directly from hosted through + * satellite-sync and only limit to supporting cloned. + * @param channel channel object + * @return compsPath comps file path + */ + public abstract String getCompsFilePath(Channel channel); + + /** + * Deletes repomd files + * @param channelLabelToProcess channel label + * @param deleteDir directory to delete + */ + public void deleteRepomdFiles(String channelLabelToProcess, boolean deleteDir) { + log.info("Removing " + channelLabelToProcess); + String prefix = mountPoint + File.separator + pathPrefix + File.separator + + channelLabelToProcess; + File primary = new File(prefix + File.separator + "primary.xml.gz"); + File filelists = new File(prefix + File.separator + "filelists.xml.gz"); + File other = new File(prefix + File.separator + "other.xml.gz"); + File repomd = new File(prefix + File.separator + "repomd.xml"); + File updateinfo = new File(prefix + File.separator + "updateinfo.xml.gz"); + File norepo = new File(prefix + File.separator + "noyumrepo.txt"); + File theDirectory = new File(prefix); + + if (!primary.delete()) { + log.info("Couldn't remove " + primary.getAbsolutePath()); + } + if (!filelists.delete()) { + log.info("Couldn't remove " + filelists.getAbsolutePath()); + } + if (!other.delete()) { + log.info("Couldn't remove " + other.getAbsolutePath()); + } + if (!repomd.delete()) { + log.info("Couldn't remove " + repomd.getAbsolutePath()); + } + updateinfo.delete(); + norepo.delete(); + if (deleteDir) { + if (!theDirectory.delete()) { + log.info("Couldn't remove " + prefix); + } + } + } } diff --git a/java/code/src/com/redhat/rhn/taskomatic/task/repomd/RpmRepositoryWriter.java b/java/code/src/com/redhat/rhn/taskomatic/task/repomd/RpmRepositoryWriter.java new file mode 100644 index 0000000..27cf51a --- /dev/null +++ b/java/code/src/com/redhat/rhn/taskomatic/task/repomd/RpmRepositoryWriter.java @@ -0,0 +1,472 @@ +/** + * Copyright (c) 2009--2010 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.rhn.taskomatic.task.repomd; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import com.redhat.rhn.common.conf.Config; +import com.redhat.rhn.common.conf.ConfigDefaults; +import com.redhat.rhn.common.hibernate.HibernateFactory; +import com.redhat.rhn.common.localization.LocalizationService; +import com.redhat.rhn.common.util.StringUtil; +import com.redhat.rhn.domain.channel.Channel; +import com.redhat.rhn.domain.channel.ChannelFactory; +import com.redhat.rhn.frontend.dto.PackageDto; +import com.redhat.rhn.manager.channel.ChannelManager; +import com.redhat.rhn.manager.rhnpackage.PackageManager; +import com.redhat.rhn.manager.task.TaskManager; + +/** +* +* @version $Rev $ +* +*/ +public class RpmRepositoryWriter extends RepositoryWriter { + + private static final String PRIMARY_FILE = "primary.xml.gz.new"; + private static final String FILELISTS_FILE = "filelists.xml.gz.new"; + private static final String OTHER_FILE = "other.xml.gz.new"; + private static final String REPOMD_FILE = "repomd.xml.new"; + private static final String UPDATEINFO_FILE = "updateinfo.xml.gz.new"; + private static final String NOREPO_FILE = "noyumrepo.txt"; + + private String checksumtype; + + /** + * Constructor takes in pathprefix and mountpoint + * @param pathPrefixIn prefix to package path + * @param mountPointIn mount point package resides + */ + public RpmRepositoryWriter(String pathPrefixIn, String mountPointIn) { + super(pathPrefixIn, mountPointIn); + } + + /** + * + * @param channel channel info + * @return repodata sanity + */ + public boolean isChannelRepodataStale(Channel channel) { + File theFile = new File(mountPoint + File.separator + pathPrefix + + File.separator + channel.getLabel() + File.separator + + "repomd.xml"); + Date fileModifiedDate = new Date(theFile.lastModified()); + // the file Modified date should be getting set when the file + // is moved into the correct location. + log.info("File Modified Date:" + LocalizationService.getInstance(). + formatCustomDate(fileModifiedDate)); + log.info("Channel Modified Date:" + LocalizationService.getInstance(). + formatCustomDate(channel.getLastModified())); + return !fileModifiedDate.equals(channel.getLastModified()); + } + + /** + * + * @param channel channelinfo for repomd file creation + */ + public void writeRepomdFiles(Channel channel) { + PackageManager.createRepoEntrys(channel.getId()); + + String prefix = mountPoint + File.separator + pathPrefix + + File.separator + channel.getLabel() + File.separator; + + // we closed the session, so we need to reload the object + channel = (Channel) HibernateFactory.getSession().get(channel.getClass(), + channel.getId()); + if (!new File(prefix).mkdirs() && !new File(prefix).exists()) { + throw new RepomdRuntimeException("Unable to create directory: " + + prefix); + } + // Get compatible checksumType + this.checksumtype = channel.getChecksumTypeLabel(); + if (checksumtype == null) { + generateBadRepo(channel, prefix); + return; + } + new File(prefix + NOREPO_FILE).delete(); + log.info("Checksum Type Value: " + this.checksumtype); + + // java.security.MessageDigest recognizes: + // MD2, MD5, SHA-1, SHA-256, SHA-384, SHA-512 + String checksumAlgo = this.checksumtype; + if (checksumAlgo.toUpperCase().startsWith("SHA")) { + checksumAlgo = this.checksumtype.substring(0, 3) + "-" + + this.checksumtype.substring(3); + } + // translate sha1 to sha for xml repo files + String checksumLabel = this.checksumtype; + if (checksumLabel.equals("sha1")) { + checksumLabel = "sha"; + } + + log.info("Generating new repository metatada for channel '" + + channel.getLabel() + "' " + channel.getPackageCount() + + " packages, " + channel.getErrataCount() + " updates"); + + CompressingDigestOutputWriter primaryFile; + CompressingDigestOutputWriter filelistsFile; + CompressingDigestOutputWriter otherFile; + + try { + primaryFile = new CompressingDigestOutputWriter( + new FileOutputStream(prefix + PRIMARY_FILE), + checksumAlgo); + filelistsFile = new CompressingDigestOutputWriter( + new FileOutputStream(prefix + FILELISTS_FILE), + checksumAlgo); + otherFile = new CompressingDigestOutputWriter( + new FileOutputStream(prefix + OTHER_FILE), checksumAlgo); + } + catch (IOException e) { + throw new RepomdRuntimeException(e); + } + catch (NoSuchAlgorithmException e) { + throw new RepomdRuntimeException(e); + } + + BufferedWriter primaryBufferedWriter = new BufferedWriter( + new OutputStreamWriter(primaryFile)); + BufferedWriter filelistsBufferedWriter = new BufferedWriter( + new OutputStreamWriter(filelistsFile)); + BufferedWriter otherBufferedWriter = new BufferedWriter( + new OutputStreamWriter(otherFile)); + PrimaryXmlWriter primary = new PrimaryXmlWriter( + primaryBufferedWriter); + FilelistsXmlWriter filelists = new FilelistsXmlWriter( + filelistsBufferedWriter); + OtherXmlWriter other = new OtherXmlWriter(otherBufferedWriter); + Date start = new Date(); + + primary.begin(channel); + filelists.begin(channel); + other.begin(channel); + + for (PackageDto pkgDto : TaskManager.getChannelPackageDtos(channel)) { + primary.addPackage(pkgDto); + filelists.addPackage(pkgDto); + other.addPackage(pkgDto); + try { + primaryFile.flush(); + filelistsFile.flush(); + otherFile.flush(); + } + catch (IOException e) { + throw new RepomdRuntimeException(e); + } + } + primary.end(); + filelists.end(); + other.end(); + try { + primaryBufferedWriter.close(); + filelistsBufferedWriter.close(); + otherBufferedWriter.close(); + } + catch (IOException e) { + throw new RepomdRuntimeException(e); + } + + RepomdIndexData primaryData = new RepomdIndexData(primaryFile + .getCompressedChecksum(), primaryFile + .getUncompressedChecksum(), channel.getLastModified()); + RepomdIndexData filelistsData = new RepomdIndexData(filelistsFile + .getCompressedChecksum(), filelistsFile + .getUncompressedChecksum(), channel.getLastModified()); + RepomdIndexData otherData = new RepomdIndexData(otherFile + .getCompressedChecksum(), otherFile + .getUncompressedChecksum(), channel.getLastModified()); + + log.info("Starting updateinfo generation for '" + channel.getLabel() + '"'); + log.info("Checksum Type Value for generate updateinfo " + this.checksumtype); + RepomdIndexData updateinfoData = generateUpdateinfo(channel, + prefix, checksumAlgo); + + RepomdIndexData groupsData = loadCompsFile(channel, checksumAlgo); + + // Set the type so yum can read and perform checksum + primaryData.setType(checksumLabel); + filelistsData.setType(checksumLabel); + otherData.setType(checksumLabel); + if (updateinfoData != null) { + updateinfoData.setType(checksumLabel); + } + + if (groupsData != null) { + groupsData.setType(checksumLabel); + } + + log.info("Primary xml's type: " + primaryData.getType()); + log.info("filelists xml's type: " + filelistsData.getType()); + log.info("other xml's type: " + otherData.getType()); + + FileWriter indexFile; + + try { + indexFile = new FileWriter(prefix + REPOMD_FILE); + } + catch (IOException e) { + throw new RepomdRuntimeException(e); + } + + RepomdIndexWriter index = new RepomdIndexWriter(indexFile, primaryData, + filelistsData, otherData, updateinfoData, groupsData); + + index.writeRepomdIndex(); + + try { + indexFile.close(); + } + catch (IOException e) { + throw new RepomdRuntimeException(e); + } + + renameFiles(prefix, channel.getLastModified().getTime(), + updateinfoData != null); + + log.info("Repository metadata generation for '" + + channel.getLabel() + "' finished in " + + (int) (new Date().getTime() - start.getTime()) / 1000 + " seconds"); + } + + /** + * Deletes existing repo and generates file stating that no repo was generated + * @param channel the channel to do this for + * @param prefix the directory prefix + */ + private void generateBadRepo(Channel channel, String prefix) { + log.warn("No repo will be generated for channel " + channel.getLabel()); + deleteRepomdFiles(channel.getLabel(), false); + try { + FileWriter norepo = new FileWriter(prefix + NOREPO_FILE); + norepo.write("No repo will be generated for channel " + + channel.getLabel() + ".\n"); + norepo.close(); + } + catch (IOException e) { + log.warn("Cannot create " + NOREPO_FILE + " file."); + } + return; + } + + private String getCompsRelativeFilename(Channel channel) { + String relativeFilename = null; + + if (channel.getComps() == null) { + relativeFilename = getCompsFilePath(channel); + } + else { + relativeFilename = channel.getComps().getRelativeFilename(); + } + + // if we didn't find anything, let's check channel's original + if (relativeFilename == null && channel.isCloned()) { + // use a hack not to use ClonedChannel and it's getOriginal() method + Long originalChannelId = ChannelManager.lookupOriginalId(channel); + Channel originalChannel = ChannelFactory.lookupById(originalChannelId); + return getCompsRelativeFilename(originalChannel); + } + + return relativeFilename; + } + + /** + * + * @param channel channel indo + * @param checksumAlgo checksum algorithm + * @return repomd index for given channel + */ + private RepomdIndexData loadCompsFile(Channel channel, String checksumAlgo) { + String compsMount = Config.get().getString(ConfigDefaults.MOUNT_POINT); + String relativeFilename = getCompsRelativeFilename(channel); + + if (relativeFilename == null) { + return null; + } + + File compsFile = new File(compsMount + File.separator + relativeFilename); + FileInputStream stream; + try { + stream = new FileInputStream(compsFile); + } + catch (FileNotFoundException e) { + return null; + } + + DigestInputStream digestStream; + try { + digestStream = new DigestInputStream(stream, MessageDigest + .getInstance(checksumAlgo)); + } + catch (NoSuchAlgorithmException nsae) { + throw new RepomdRuntimeException(nsae); + } + byte[] bytes = new byte[10]; + + try { + while (digestStream.read(bytes) != -1) { + // no-op + } + } + catch (IOException e) { + return null; + } + + Date timeStamp = new Date(compsFile.lastModified()); + + return new RepomdIndexData(StringUtil.getHexString(digestStream + .getMessageDigest().digest()), null, timeStamp); + } + + /** + * TODO: This static comps paths should go away once + * we can get the paths directly from hosted through + * satellite-sync and only limit to supporting cloned. + * @param channel channel object + * @return compsPath comps file path + */ + public String getCompsFilePath(Channel channel) { + String compsPath = null; + + Map<String, String> compsMapping = new HashMap<String, String>(); + String rootClientPath = "/rhn/kickstart/ks-rhel-x86_64-client-5"; + String rootServerPath = "/rhn/kickstart/ks-rhel-x86_64-server-5"; + compsMapping.put("rhel-x86_64-client-5", + rootClientPath + "/Client/repodata/comps-rhel5-client-core.xml"); + compsMapping.put("rhel-x86_64-client-vt-5", + rootClientPath + "/VT/repodata/comps-rhel5-vt.xml"); + compsMapping.put("rhel-x86_64-client-workstation-5", + rootClientPath + "/Workstation/repodata/comps-rhel5-client-workstation.xml"); + compsMapping.put("rhel-x86_64-server-5", + rootServerPath + "/Server/repodata/comps-rhel5-server-core.xml"); + compsMapping.put("rhel-x86_64-server-vt-5", + rootServerPath + "/VT/repodata/comps-rhel5-vt.xml"); + compsMapping.put("rhel-x86_64-server-cluster-5", + rootServerPath + "/Cluster/repodata/comps-rhel5-cluster.xml"); + compsMapping.put("rhel-x86_64-server-cluster-storage-5", + rootServerPath + "/ClusterStorage/repodata/comps-rhel5-cluster-st.xml"); + + String[] arches = {"i386", "ia64", "s390x", "ppc"}; + Map<String, String> newCompsmap = new HashMap<String, String>(); + for (String k : compsMapping.keySet()) { + for (String arch : arches) { + newCompsmap.put(k.replace("x86_64", arch), + compsMapping.get(k).replace("x86_64", arch)); + } + } + compsMapping.putAll(newCompsmap); + + if (compsMapping.containsKey(channel.getLabel())) { + compsPath = compsMapping.get(channel.getLabel()); + } + + return compsPath; + } + + /** + * Generates update info for given channel + * @param channel channel info + * @param prefix repodata file prefix + * @param checksumtype checksum type + * @return repodata index + */ + private RepomdIndexData generateUpdateinfo(Channel channel, String prefix, + String checksumtypeIn) { + + if (channel.getErrataCount() == 0) { + return null; + } + + CompressingDigestOutputWriter updateinfoFile; + try { + updateinfoFile = new CompressingDigestOutputWriter( + new FileOutputStream(prefix + UPDATEINFO_FILE), checksumtypeIn); + } + catch (FileNotFoundException e) { + throw new RepomdRuntimeException(e); + } + catch (NoSuchAlgorithmException e) { + throw new RepomdRuntimeException(e); + } + catch (IOException e) { + throw new RepomdRuntimeException(e); + } + BufferedWriter updateinfoBufferedWriter = new BufferedWriter( + new OutputStreamWriter(updateinfoFile)); + UpdateInfoWriter updateinfo = new UpdateInfoWriter( + updateinfoBufferedWriter); + updateinfo.getUpdateInfo(channel); + try { + updateinfoBufferedWriter.close(); + } + catch (IOException e) { + throw new RepomdRuntimeException(e); + } + + RepomdIndexData updateinfoData = new RepomdIndexData(updateinfoFile + .getCompressedChecksum(), updateinfoFile + .getUncompressedChecksum(), channel.getLastModified()); + return updateinfoData; + } + + /** + * Renames the repo cache files + * @param prefix path prefix + * @param lastModified file last_modified + * @param doUpdateinfo + */ + private void renameFiles(String prefix, Long lastModified, + Boolean doUpdateinfo) { + File primary = new File(prefix + PRIMARY_FILE); + File filelists = new File(prefix + FILELISTS_FILE); + File other = new File(prefix + OTHER_FILE); + File repomd = new File(prefix + REPOMD_FILE); + + File updateinfo = null; + if (doUpdateinfo) { + updateinfo = new File(prefix + UPDATEINFO_FILE); + } + + if (doUpdateinfo) { + updateinfo.setLastModified(lastModified); + } + + primary.setLastModified(lastModified); + filelists.setLastModified(lastModified); + other.setLastModified(lastModified); + repomd.setLastModified(lastModified); + + if (doUpdateinfo) { + updateinfo.renameTo(new File(prefix + "updateinfo.xml.gz")); + } + + primary.renameTo(new File(prefix + "primary.xml.gz")); + filelists.renameTo(new File(prefix + "filelists.xml.gz")); + other.renameTo(new File(prefix + "other.xml.gz")); + repomd.renameTo(new File(prefix + "repomd.xml")); + } +}
commit 1134ca5bde45c3f62de5b9340d8c32dcab5af3e4 Author: Miroslav Suchý msuchy@redhat.com Date: Wed Oct 6 10:03:35 2010 +0200
correct Summary and Description
diff --git a/client/tools/spacewalk-remote-utils/spacewalk-remote-utils.spec b/client/tools/spacewalk-remote-utils/spacewalk-remote-utils.spec index 06b15f8..454b69f 100644 --- a/client/tools/spacewalk-remote-utils/spacewalk-remote-utils.spec +++ b/client/tools/spacewalk-remote-utils/spacewalk-remote-utils.spec @@ -6,7 +6,7 @@ Name: spacewalk-remote-utils Version: 1.0.4 Release: 1%{?dist} -Summary: Utilities to interact with a Satellite or Spacewalk server remotely. +Summary: Utilities to interact with a RHN Satellite or Spacewalk server remotely.
Group: Applications/System License: GPLv3+ @@ -19,7 +19,7 @@ BuildRequires: python-devel BuildRequires: docbook-utils
%description -Utilities to interact with a Satellite or Spacewalk server remotely over XMLRPC. +Utilities to interact with a RHN Satellite or Spacewalk server remotely over XMLRPC.
%prep %setup -q
commit f6a84fd58b116d2311b932fea111ff3082de348d Author: Miroslav Suchý msuchy@redhat.com Date: Wed Oct 6 10:01:47 2010 +0200
correct URL and Source0
diff --git a/client/tools/spacewalk-remote-utils/spacewalk-remote-utils.spec b/client/tools/spacewalk-remote-utils/spacewalk-remote-utils.spec index 1849f83..06b15f8 100644 --- a/client/tools/spacewalk-remote-utils/spacewalk-remote-utils.spec +++ b/client/tools/spacewalk-remote-utils/spacewalk-remote-utils.spec @@ -10,8 +10,8 @@ Summary: Utilities to interact with a Satellite or Spacewalk server remotely
Group: Applications/System License: GPLv3+ -URL: https://fedorahosted.org/spacewalk/wiki/spacecmd -Source: %{name}-%{version}.tar.gz +URL: http://fedorahosted.org/spacewalk +Source: https://fedorahosted.org/releases/s/p/spacewalk/%%7Bname%7D-%%7Bversion%7D.t... BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) BuildArch: noarch