backend/satellite_tools/repo_plugins/yum_src.py | 56 ++ backend/satellite_tools/reposync.py | 201 ++++++++++ backend/satellite_tools/spacewalk-repo-sync | 89 ++++ backend/server/basePackageUpload.py | 100 ++++ backend/server/rhnPackage.py | 16 backend/server/rhnPackageUpload.py | 71 --- backend/upload_server/handlers/package_push/package_push.py | 10 backend/upload_server/handlers/package_upload/package_upload.py | 8 java/code/src/com/redhat/rhn/domain/channel/Channel.java | 7 java/code/src/com/redhat/rhn/domain/channel/ChannelFactory.java | 13 java/code/src/com/redhat/rhn/domain/channel/ContentSource.hbm.xml | 7 java/code/src/com/redhat/rhn/domain/channel/ContentSource.java | 17 java/code/src/com/redhat/rhn/domain/task/Task.hbm.xml | 8 java/code/src/com/redhat/rhn/domain/task/TaskFactory.java | 29 + java/code/src/com/redhat/rhn/frontend/action/channel/manage/EditChannelAction.java | 4 java/code/src/com/redhat/rhn/frontend/strings/jsp/StringResource_en_US.xml | 6 java/code/src/com/redhat/rhn/manager/channel/CreateChannelCommand.java | 19 java/code/src/com/redhat/rhn/manager/channel/UpdateChannelCommand.java | 2 java/code/src/com/redhat/rhn/taskomatic/task/RepoSyncTask.java | 94 ++++ java/code/webapp/WEB-INF/pages/channel/manage/edit.jsp | 10 java/code/webapp/WEB-INF/struts-config.xml | 1 schema/spacewalk/rhnsat/tables/rhnChannelContentSource.sql | 2 schema/spacewalk/upgrade/spacewalk-schema-0.5-to-spacewalk-schema-0.6/192-rhnChannelContentSoource.sql | 2 23 files changed, 687 insertions(+), 85 deletions(-)
New commits: commit 6ed3ca255562fef25eb51bcb2523a2929af504a7 Author: Justin Sherrill jsherril@redhat.com Date: Wed Jul 22 18:24:16 2009 -0400
adding repo sync task and other UI bits for spacewalk repo sync
diff --git a/backend/satellite_tools/repo_plugins/yum_src.py b/backend/satellite_tools/repo_plugins/yum_src.py new file mode 100644 index 0000000..fd39518 --- /dev/null +++ b/backend/satellite_tools/repo_plugins/yum_src.py @@ -0,0 +1,56 @@ +#!/usr/bin/python -u +# +# Copyright (c) 2008 Red Hat, Inc. +# +# This software is licensed to you under the GNU General Public License, +# version 2 (GPLv2). There is NO WARRANTY for this software, express or +# implied, including the implied warranties of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 +# along with this software; if not, see +# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. +# +# Red Hat trademarks are not licensed under GPLv2. No permission is +# granted to use or replicate Red Hat trademarks that are incorporated +# in this software or its documentation. +# +import yum +from yum import config +from reposync import ContentPackage +class ContentSource: + + url = None + name = None + repo = None + + def __init__(self, url, name): + self.url = url + self.name = name + + def list_packages(self): + """ list packages""" + repo = yum.yumRepo.YumRepository(self.name) + self.repo = repo + repo.cache = 0 + repo.metadata_expire = 5 + repo.baseurl = [self.url] + repo.basecachedir = '/var/cache/rhn/reposync/' +# repo.cachedir = '/var/cache/rhn/reposync/' + repo.setup(False) + sack = repo.getPackageSack() + sack.populate(repo, 'metadata', None, 0) + list = sack.returnPackages() + to_return = [] + for pack in list: + new_pack = ContentPackage() + new_pack.setNVREA(pack.name, pack.version, pack.release, pack.epoch, pack.arch) + new_pack.unique_id = pack + for cs in pack.checksums: + new_pack.checksums[cs[0]] = cs[1] + to_return.append(new_pack) + return to_return + + def get_package(self, package): + """ get package """ + return self.repo.getPackage(package.unique_id) + + diff --git a/java/code/src/com/redhat/rhn/domain/channel/Channel.java b/java/code/src/com/redhat/rhn/domain/channel/Channel.java index 303fa7d..e61da3f 100644 --- a/java/code/src/com/redhat/rhn/domain/channel/Channel.java +++ b/java/code/src/com/redhat/rhn/domain/channel/Channel.java @@ -819,7 +819,7 @@ public class Channel extends BaseDomainHelper implements Comparable { * if null or '' is passed in, all content sources will be removed. * @param url the url of the yum repo */ - public void setYumContentSource(String url) { + public void setYumContentSource(String url, String labelIn) { if (StringUtils.isEmpty(url)) { if (!this.getContentSources().isEmpty()) { this.getContentSources().clear(); @@ -830,11 +830,14 @@ public class Channel extends BaseDomainHelper implements Comparable { ContentSource cs = new ContentSource(); cs.setChannel(this); cs.setSourceUrl(url); + cs.setLabel(labelIn); cs.setType(ChannelFactory.CONTENT_SOURCE_TYPE_YUM); this.getContentSources().add(cs); } else { - this.getContentSources().iterator().next().setSourceUrl(url); + ContentSource cs = this.getContentSources().iterator().next(); + cs.setSourceUrl(url); + cs.setLabel(labelIn); } } ChannelFactory.save(this); diff --git a/java/code/src/com/redhat/rhn/domain/channel/ChannelFactory.java b/java/code/src/com/redhat/rhn/domain/channel/ChannelFactory.java index 7f3aa2a..71a73b1 100644 --- a/java/code/src/com/redhat/rhn/domain/channel/ChannelFactory.java +++ b/java/code/src/com/redhat/rhn/domain/channel/ChannelFactory.java @@ -120,6 +120,19 @@ public class ChannelFactory extends HibernateFactory { }
/** + * Lookup a content source by id + * @param id the id to lookup + * @return the ContentSource + */ + public static ContentSource lookupContentSource(Long id) { + Map params = new HashMap(); + params.put("id", id); + return (ContentSource) singleton.lookupObjectByNamedQuery( + "ContentSource.findById", params); + } + + + /** * Retrieve a list of channel ids associated with the labels provided * @param labelsIn the labels to search for * @return list of channel ids diff --git a/java/code/src/com/redhat/rhn/domain/channel/ContentSource.hbm.xml b/java/code/src/com/redhat/rhn/domain/channel/ContentSource.hbm.xml index 2756941..fd8edd2 100644 --- a/java/code/src/com/redhat/rhn/domain/channel/ContentSource.hbm.xml +++ b/java/code/src/com/redhat/rhn/domain/channel/ContentSource.hbm.xml @@ -17,6 +17,7 @@ PUBLIC "-//Hibernate/Hibernate Mapping DTD 3.0//EN" <property name="lastSynced" type="timestamp" column="last_synced"/> <property name="created" type="timestamp" column="created" insert="false" update="false" /> <property name="modified" type="timestamp" column="modified" insert="false" update="false" /> + <property name="label" type="string" column="label"/>
<many-to-one @@ -31,4 +32,10 @@ PUBLIC "-//Hibernate/Hibernate Mapping DTD 3.0//EN"
</class> + + + <query name="ContentSource.findById"> + <![CDATA[from com.redhat.rhn.domain.channel.ContentSource as c where c.id = :id]]> + </query> + </hibernate-mapping> diff --git a/java/code/src/com/redhat/rhn/domain/channel/ContentSource.java b/java/code/src/com/redhat/rhn/domain/channel/ContentSource.java index 70a54e8..74599a1 100644 --- a/java/code/src/com/redhat/rhn/domain/channel/ContentSource.java +++ b/java/code/src/com/redhat/rhn/domain/channel/ContentSource.java @@ -29,8 +29,25 @@ public class ContentSource extends BaseDomainHelper { private ContentSourceType type; private String sourceUrl; private Date lastSynced; + private String label;
+ + /** + * @return Returns the label. + */ + public String getLabel() { + return label; + } + + + /** + * @param labelIn The label to set. + */ + public void setLabel(String labelIn) { + this.label = labelIn; + } + /** * @return Returns the id. */ diff --git a/java/code/src/com/redhat/rhn/domain/task/Task.hbm.xml b/java/code/src/com/redhat/rhn/domain/task/Task.hbm.xml index 276b5e8..70b8f1a 100644 --- a/java/code/src/com/redhat/rhn/domain/task/Task.hbm.xml +++ b/java/code/src/com/redhat/rhn/domain/task/Task.hbm.xml @@ -32,6 +32,14 @@ PUBLIC "-//Hibernate/Hibernate Mapping DTD 3.0//EN" where org_id = :org_id and task_name = :name and task_data = :data and earliest = :earliest]]> </query> + + <query name="Task.listTasksByName"> + <![CDATA[from com.redhat.rhn.domain.task.Task as t + where t.name = :name]]> + </query> + + +
<sql-query name="Task.lookupByNameLike"> <![CDATA[SELECT {t.*} diff --git a/java/code/src/com/redhat/rhn/domain/task/TaskFactory.java b/java/code/src/com/redhat/rhn/domain/task/TaskFactory.java index 50149eb..aa75626 100644 --- a/java/code/src/com/redhat/rhn/domain/task/TaskFactory.java +++ b/java/code/src/com/redhat/rhn/domain/task/TaskFactory.java @@ -19,10 +19,12 @@ import com.redhat.rhn.domain.org.Org;
import org.apache.log4j.Logger; import org.hibernate.Criteria; +import org.hibernate.HibernateException; import org.hibernate.Session; import org.hibernate.criterion.CriteriaSpecification; import org.hibernate.criterion.Order;
+import java.util.Collections; import java.util.Date; import java.util.List;
@@ -66,6 +68,33 @@ public class TaskFactory extends HibernateFactory { }
/** + * + * @param name + * @return + */ + public static List<Task> listTasks(String name) { + Session session = null; + try { + session = HibernateFactory.getSession(); + return session.getNamedQuery("Task.listTasksByName") + .setString("name", name).list(); + } + catch (HibernateException he) { + log.error("Hibernate exception: " + he.toString()); + } + return Collections.EMPTY_LIST; + } + + /** + * + * @param task + */ + public static void removeTask(Task task) { + TaskFactory.getSession().delete(task); + } + + + /** * Saves the object to the db * @param taskIn The task to save */ diff --git a/java/code/src/com/redhat/rhn/frontend/action/channel/manage/EditChannelAction.java b/java/code/src/com/redhat/rhn/frontend/action/channel/manage/EditChannelAction.java index c7c8f66..baa0737 100644 --- a/java/code/src/com/redhat/rhn/frontend/action/channel/manage/EditChannelAction.java +++ b/java/code/src/com/redhat/rhn/frontend/action/channel/manage/EditChannelAction.java @@ -334,6 +334,7 @@ public class EditChannelAction extends RhnAction implements Listable { ucc.setSupportPolicy((String)form.get("support_policy")); ucc.setAccess((String)form.get("org_sharing")); ucc.setYumUrl((String) form.getString("yum_repo")); + ucc.setRepoLabel((String) form.getString("repo_label"));
String parent = (String)form.get("parent"); if (parent == null || parent.equals("")) { @@ -405,6 +406,7 @@ public class EditChannelAction extends RhnAction implements Listable { ccc.setSupportPolicy((String)form.get("support_policy")); ccc.setAccess((String)form.get("org_sharing")); ccc.setYumUrl((String) form.getString("yum_repo")); + ccc.setRepoLabel((String) form.getString("repo_label"));
String parent = (String)form.get("parent"); if (parent == null || parent.equals("")) { @@ -570,11 +572,13 @@ public class EditChannelAction extends RhnAction implements Listable {
if (c.getContentSources().isEmpty()) { form.set("yum_repo", ""); + form.set("repo_label", ""); request.setAttribute("last_sync", ""); } else { ContentSource cs = c.getContentSources().iterator().next(); form.set("yum_repo", cs.getSourceUrl()); + form.set("repo_label", cs.getLabel()); String lastSync = LocalizationService.getInstance().getMessage( "channel.edit.repo.neversynced"); if (cs.getLastSynced() != null) { diff --git a/java/code/src/com/redhat/rhn/frontend/strings/jsp/StringResource_en_US.xml b/java/code/src/com/redhat/rhn/frontend/strings/jsp/StringResource_en_US.xml index 8c76681..82aabc0 100644 --- a/java/code/src/com/redhat/rhn/frontend/strings/jsp/StringResource_en_US.xml +++ b/java/code/src/com/redhat/rhn/frontend/strings/jsp/StringResource_en_US.xml @@ -21431,6 +21431,12 @@ given channel.</source> <trans-unit id="channel.edit.jsp.repourl"> <source>Repository URL</source> </trans-unit> + <trans-unit id="channel.edit.jsp.repolabel"> + <source>Label</source> + </trans-unit> + <trans-unit id="edit.channel.invalidrepolabel.missing"> + <source>A repo label is required if providing a yum repository url.</source> + </trans-unit> <trans-unit id="channel.edit.jsp.lastsynced"> <source>Last Sync Time</source> </trans-unit> diff --git a/java/code/src/com/redhat/rhn/manager/channel/CreateChannelCommand.java b/java/code/src/com/redhat/rhn/manager/channel/CreateChannelCommand.java index d75225c..0ea6469 100644 --- a/java/code/src/com/redhat/rhn/manager/channel/CreateChannelCommand.java +++ b/java/code/src/com/redhat/rhn/manager/channel/CreateChannelCommand.java @@ -68,9 +68,12 @@ public class CreateChannelCommand { protected String supportPolicy; protected String access = Channel.PRIVATE; protected String yumUrl; + protected String repoLabel;
+
+ /** * default constructor. */ @@ -269,7 +272,8 @@ public class CreateChannelCommand {
ChannelFactory.refreshNewestPackageCache(c, WEB_CHANNEL_CREATED);
- c.setYumContentSource(yumUrl); + c.setYumContentSource(yumUrl, repoLabel); +
return c; } @@ -324,6 +328,12 @@ public class CreateChannelCommand { throw new IllegalArgumentException( "Required parameters not set: user, or summary"); } + + if (!StringUtils.isEmpty(yumUrl) && StringUtils.isEmpty(repoLabel)) { + throw new IllegalArgumentException( + "edit.channel.invalidrepolabel.missing"); + } + }
protected void verifyChannelName(String cname) throws InvalidChannelNameException { @@ -432,5 +442,12 @@ public class CreateChannelCommand { public void setYumUrl(String yumUrlIn) { this.yumUrl = yumUrlIn; } + + /** + * @param repoLabelIn The repoLabel to set. + */ + public void setRepoLabel(String repoLabelIn) { + this.repoLabel = repoLabelIn; + }
} diff --git a/java/code/src/com/redhat/rhn/manager/channel/UpdateChannelCommand.java b/java/code/src/com/redhat/rhn/manager/channel/UpdateChannelCommand.java index 66d8673..79764aa 100644 --- a/java/code/src/com/redhat/rhn/manager/channel/UpdateChannelCommand.java +++ b/java/code/src/com/redhat/rhn/manager/channel/UpdateChannelCommand.java @@ -79,7 +79,7 @@ public class UpdateChannelCommand extends CreateChannelCommand { c.setSupportPolicy(supportPolicy);
- c.setYumContentSource(yumUrl); + c.setYumContentSource(yumUrl, repoLabel);
// need to save before calling stored proc below diff --git a/java/code/src/com/redhat/rhn/taskomatic/task/RepoSyncTask.java b/java/code/src/com/redhat/rhn/taskomatic/task/RepoSyncTask.java new file mode 100644 index 0000000..ae382a5 --- /dev/null +++ b/java/code/src/com/redhat/rhn/taskomatic/task/RepoSyncTask.java @@ -0,0 +1,94 @@ +/** + * Copyright (c) 2009 Red Hat, Inc. + * + * This software is licensed to you under the GNU General Public License, + * version 2 (GPLv2). There is NO WARRANTY for this software, express or + * implied, including the implied warranties of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 + * along with this software; if not, see + * http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. + * + * Red Hat trademarks are not licensed under GPLv2. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.rhn.taskomatic.task; + +import com.redhat.rhn.domain.channel.ChannelFactory; +import com.redhat.rhn.domain.channel.ContentSource; +import com.redhat.rhn.domain.task.Task; +import com.redhat.rhn.domain.task.TaskFactory; + +import org.apache.log4j.Logger; +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/** + * Repo Sync + * Used for syncing repos (like yum repos) to a channel + * This really just calls a python script + * + * @version $Rev$ + */ +public class RepoSyncTask implements Job { + + /** + * Used to log stats in the RHNDAEMONSTATE table + */ + public static final String DISPLAY_NAME = "repo_sync"; + + private static Logger log = Logger.getLogger(RepoSyncTask.class); + + /** + * Default constructor + */ + public RepoSyncTask() { + } + + + /** + * + * {@inheritDoc} + */ + public void execute(JobExecutionContext context) + throws JobExecutionException { + + for (Task task : TaskFactory.listTasks("repo_sync")) { + + ContentSource src = ChannelFactory.lookupContentSource(task.getData()); + if (src == null) { + log.error("Content Source could not be found: " + task.getData()); + continue; + } + + try { + Process p = Runtime.getRuntime().exec(""); + } + catch (IOException e) { + + } + src.setLastSynced(new Date()); + TaskFactory.removeTask(task); + } + } + + private static List<String> getSyncCommand(ContentSource src) { + List<String> cmd = new ArrayList<String>(); + cmd.add("/usr/sbin/spacewalk-repo-sync"); + cmd.add("-c"); + cmd.add(src.getChannel().getLabel()); + cmd.add("-u"); + cmd.add(src.getSourceUrl()); + cmd.add("-t"); + cmd.add(src.getType().getLabel()); + return cmd; + } + + +} diff --git a/java/code/webapp/WEB-INF/pages/channel/manage/edit.jsp b/java/code/webapp/WEB-INF/pages/channel/manage/edit.jsp index eb7649a..adda2fb 100644 --- a/java/code/webapp/WEB-INF/pages/channel/manage/edit.jsp +++ b/java/code/webapp/WEB-INF/pages/channel/manage/edit.jsp @@ -117,7 +117,7 @@ <bean:message key="channel.edit.jsp.repourl"/>: </th> <td> - <html:text property="yum_repo" maxlength="512" size="60" /> + <html:text property="yum_repo" maxlength="512" size="64" /> </td> </tr> <c:if test='${not empty param.cid}'> @@ -132,6 +132,14 @@ </c:if> <tr> <th> + <bean:message key="channel.edit.jsp.repolabel"/>: + </th> + <td> + <html:text property="repo_label" maxlength="64" size="20" /> + </td> + </tr> + <tr> + <th> <bean:message key="channel.edit.jsp.sync"/>: </th> <td> diff --git a/java/code/webapp/WEB-INF/struts-config.xml b/java/code/webapp/WEB-INF/struts-config.xml index d09d20d..8086357 100644 --- a/java/code/webapp/WEB-INF/struts-config.xml +++ b/java/code/webapp/WEB-INF/struts-config.xml @@ -838,6 +838,7 @@ <form-property name="maintainer_phone" type="java.lang.String" /> <form-property name="support_policy" type="java.lang.String" /> <form-property name="yum_repo" type="java.lang.String" /> + <form-property name="repo_label" type="java.lang.String" /> <form-property name="sync_repo" type="java.lang.Boolean" /> <form-property name="per_user_subscriptions" type="java.lang.String" /> <form-property name="org_sharing" type="java.lang.String" />
commit 7363b853cd74456f03b24bfceb0478a89031ceb6 Author: Justin Sherrill jsherril@redhat.com Date: Wed Jul 22 18:23:56 2009 -0400
adding label to ContentSource schema
diff --git a/schema/spacewalk/rhnsat/tables/rhnChannelContentSource.sql b/schema/spacewalk/rhnsat/tables/rhnChannelContentSource.sql index 26c3973..f0b30a5 100644 --- a/schema/spacewalk/rhnsat/tables/rhnChannelContentSource.sql +++ b/schema/spacewalk/rhnsat/tables/rhnChannelContentSource.sql @@ -32,6 +32,8 @@ rhnChannelContentSource references rhnContentSourceType(id), source_url varchar2(512) constraint rhn_ccs_url_nn not null, + label varchar2(64) + constraint rhn_ccs_l_nn not null, last_synced date, created date default(sysdate) constraint rhn_ccs_cre_nn not null, diff --git a/schema/spacewalk/upgrade/spacewalk-schema-0.5-to-spacewalk-schema-0.6/192-rhnChannelContentSoource.sql b/schema/spacewalk/upgrade/spacewalk-schema-0.5-to-spacewalk-schema-0.6/192-rhnChannelContentSoource.sql index a1445e1..ccf5cb8 100644 --- a/schema/spacewalk/upgrade/spacewalk-schema-0.5-to-spacewalk-schema-0.6/192-rhnChannelContentSoource.sql +++ b/schema/spacewalk/upgrade/spacewalk-schema-0.5-to-spacewalk-schema-0.6/192-rhnChannelContentSoource.sql @@ -32,6 +32,8 @@ rhnChannelContentSource references rhnContentSourceType(id), source_url varchar2(512) constraint rhn_ccs_url_nn not null, + label varchar2(64) + constraint rhn_ccs_l_nn not null, last_synced date, created date default(sysdate) constraint rhn_ccs_cre_nn not null,
commit 7a0f31f3dabe9c0c0f226421321c58f2c8c3c29b Author: Justin Sherrill jsherril@redhat.com Date: Wed Jul 22 18:22:37 2009 -0400
backend/satellite_tools/repo_plugins/yum_src.py
initial commit of spacewalk repo sync
diff --git a/backend/satellite_tools/reposync.py b/backend/satellite_tools/reposync.py new file mode 100644 index 0000000..5abcc1c --- /dev/null +++ b/backend/satellite_tools/reposync.py @@ -0,0 +1,201 @@ +#!/usr/bin/python -u +# +# Copyright (c) 2008 Red Hat, Inc. +# +# This software is licensed to you under the GNU General Public License, +# version 2 (GPLv2). There is NO WARRANTY for this software, express or +# implied, including the implied warranties of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 +# along with this software; if not, see +# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. +# +# Red Hat trademarks are not licensed under GPLv2. No permission is +# granted to use or replicate Red Hat trademarks that are incorporated +# in this software or its documentation. +# +import sys, os +from optparse import OptionParser +from common import rhnLib +from server import rhnPackage, rhnSQL, rhnChannel, rhnPackageUpload +from common import CFG, initCFG, rhn_rpm +from server.importlib.importLib import IncompletePackage +from server.importlib.backendOracle import OracleBackend +from server.importlib.packageImport import ChannelPackageSubscription + +class RepoSync: + + + parser = None + type = None + url = None + channel_label = None + plugin = None + channel = None + fail = False + repo_label = None + + + def main(self): + initCFG('server') + db_string = CFG.DEFAULT_DB #"rhnsat/rhnsat@rhnsat" + rhnSQL.initDB(db_string) + (options, args) = self.process_args() + + + if not options.url: + print("--url must be specified") + if not options.type: + print("--type must be specified") + if not options.channel_label: + print("--channel must be specified") + if not options.label: + print("--label must be specified") + + + self.type = options.type + self.url = options.url + self.channel_label = options.channel_label + self.fail = options.fail + self.repo_label = options.label + self.channel = self.load_channel() + + if not self.channel or not rhnChannel.isCustomChannel(self.channel['id']): + print "Channel does not exist or is not custom" + sys.exit(1) + + self.plugin = self.load_plugin()(self.url, self.channel_label + "-" + self.repo_label) + self.import_packages(self.plugin.list_packages()) + + + + def process_args(self): + self.parser = OptionParser() + self.parser.add_option('-u', '--url', action='store', dest='url', help='The url to sync') + self.parser.add_option('-c', '--channel', action='store', dest='channel_label', help='The label of the channel to sync packages to') + self.parser.add_option('-t', '--type', action='store', dest='type', help='The type of repo, currently only "yum" is supported') + self.parser.add_option('-l', '--label', action='store', dest='label', help='A friendly label to refer to the repo') + self.parser.add_option('-f', '--fail', action='store_true', dest='fail', default=False , help="If a package import fails, fail the entire operation") + return self.parser.parse_args() + + + def load_plugin(self): + sys.path.append("repo_plugins") + name = self.type + "_src" + mod = __import__(name) + return getattr(mod, "ContentSource") + + def import_packages(self, packages): + to_link = [] + to_download = [] + for pack in packages: + pid = None + if pack.checksums.has_key('md5sum'): + """lookup by md5sum""" + elif pack.checksums.has_key('sha256'): + """lookup by sha256""" + if pid == None: + if self.channel_label not in rhnPackage.get_channels_for_package([pack.name, pack.version, pack.release, pack.epoch, pack.arch]) and \ + self.channel_label not in rhnPackage.get_channels_for_package([pack.name, pack.version, pack.release, '', pack.arch]): + to_download.append(pack) + + + for (index, pack) in enumerate(to_download): + """download each package""" + try: + print(str(index) + "/" + str(len(to_download)) + " : "+ pack.getNVREA()) + path = self.plugin.get_package(pack) + md5 = rhnLib.getFileMD5(filename=path) + pid = rhnPackage.get_package_for_md5sum(self.channel['org_id'], md5) + if pid is None: + self.upload_package(pack, path, md5) + self.associate_package(pack, md5) + else: + to_link.append(pack) + except KeyboardInterrupt: + raise + except: + if self.fail: + raise + continue + for pack in to_link: + try: + self.associate_package(pack, md5) + except KeyboardInterrupt: + raise + except: + if self.fail: + raise + continue + + def upload_package(self, package, path, md5): + temp_file = open(path, 'rb') + header, payload_stream, md5sum, header_start, header_end = \ + rhnPackageUpload.load_package(temp_file) + rel_package_path = rhnPackageUpload.relative_path_from_header( + header, org_id=self.channel['org_id'], md5sum=md5sum) + package_path = os.path.join(CFG.MOUNT_POINT, + rel_package_path) + package_dict, diff_level = rhnPackageUpload.push_package(header, + payload_stream, md5sum, force=False, + header_start=header_start, header_end=header_end, + relative_path=rel_package_path, org_id=self.channel['org_id']) + temp_file.close() + if self.url.find("file://") < 0: + os.remove(path) + + + def associate_package(self, pack, md5sum): + caller = "server.app.yumreposync" + backend = OracleBackend() + backend.init() + package = {} + package['name'] = pack.name + package['version'] = pack.version + package['release'] = pack.release + if pack.epoch == 0: + package['epoch'] = "" + else: + package['epoch'] = pack.epoch + package['arch'] = pack.arch + package['md5sum'] = md5sum + package['channels'] = [{'label':self.channel_label, 'id':self.channel['id']}] + package['org_id'] = self.channel['org_id'] + + try: + importer = ChannelPackageSubscription([IncompletePackage().populate(package)], backend, caller=caller) + importer.run() + except: + package['epoch'] = '0' + importer = ChannelPackageSubscription([IncompletePackage().populate(package)], backend, caller=caller) + importer.run() + backend.commit() + def load_channel(self): + return rhnChannel.channel_info(self.channel_label) + + +class ContentPackage: + + #map of checksums. Valid keys are 'md5sum' & 'sha256' + checksums = {} + + #unique ID that can be used by plugin + unique_id = None + + name = None + version = None + release = None + epoch = None + arch = None + + def setNVREA(self, name, version, release, epoch, arch): + self.name = name + self.version = version + self.release = release + self.arch = arch + + def getNVREA(self): + if self.epoch: + return self.name + '-' + self.version + '-' + self.release + '-' + self.epoch + '.' + self.arch + else: + return self.name + '-' + self.version + '-' + self.release + '.' + self.arch + diff --git a/backend/satellite_tools/spacewalk-repo-sync b/backend/satellite_tools/spacewalk-repo-sync new file mode 100755 index 0000000..244c260 --- /dev/null +++ b/backend/satellite_tools/spacewalk-repo-sync @@ -0,0 +1,89 @@ +#!/usr/bin/python -u +# +# Copyright (c) 2008 Red Hat, Inc. +# +# This software is licensed to you under the GNU General Public License, +# version 2 (GPLv2). There is NO WARRANTY for this software, express or +# implied, including the implied warranties of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 +# along with this software; if not, see +# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. +# +# Red Hat trademarks are not licensed under GPLv2. No permission is +# granted to use or replicate Red Hat trademarks that are incorporated +# in this software or its documentation. +# + +LOCK = None + +if __name__ != '__main__': + raise ImportError, "module cannot be imported" + +import sys +def systemExit(code, msgs=None): + "Exit with a code and optional message(s). Saved a few lines of code." + + if msgs: + if type(msgs) not in [type([]), type(())]: + msgs = (msgs, ) + for msg in msgs: + sys.stderr.write(str(msg)+'\n') + sys.exit(code) + + +try: + import os + import socket + _LIBPATH = "/usr/share/rhn" + if _LIBPATH not in sys.path: + sys.path.append(_LIBPATH) + from common import rhnLockfile, CFG, fetchTraceback + from satellite_tools import reposync +except KeyboardInterrupt: + systemExit(0, "\nUser interrupted process.") +except ImportError: + sys.stderr.write("Unable to find RHN code tree.\n" + "Path not correct? '%s'\n" % _LIBPATH) + + + + +def releaseLOCK(): + global LOCK + if LOCK: + LOCK.release() + LOCK = None + + +def main(): + + # quick check to see if you are a super-user. + if os.getuid() != 0: + sys.stderr.write('ERROR: must be root to execute\n') + sys.exit(8) + + + LOCK = None + try: + LOCK = rhnLockfile.Lockfile('/var/run/spacewalk-sync.pid') + except rhnLockfile.LockfileLockedException: + systemExit(1, "SYNC ERROR: attempting to run more than one instance of satellite-sync. Exiting.") + + sync = reposync.RepoSync() + sync.main() + releaseLOCK() + return 0 + + +if __name__ == '__main__': + try: + sys.exit(abs(main() or 0)) + except KeyboardInterrupt: + systemExit(0, "\nUser interrupted process.") + except SystemExit, e: + releaseLOCK() + sys.exit(e.code) + except Exception, e: + releaseLOCK() + raise + diff --git a/backend/server/basePackageUpload.py b/backend/server/basePackageUpload.py new file mode 100644 index 0000000..ad05f93 --- /dev/null +++ b/backend/server/basePackageUpload.py @@ -0,0 +1,100 @@ +#!/usr/bin/python +# +# Copyright (c) 2008 Red Hat, Inc. +# +# This software is licensed to you under the GNU General Public License, +# version 2 (GPLv2). There is NO WARRANTY for this software, express or +# implied, including the implied warranties of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 +# along with this software; if not, see +# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. +# +# Red Hat trademarks are not licensed under GPLv2. No permission is +# granted to use or replicate Red Hat trademarks that are incorporated +# in this software or its documentation. +# +# $Id$ + +import os +import tempfile +from mod_python import apache + +from common import CFG, log_debug, rhnFault, rhn_mpm, rhnLib, UserDictCase +from common.rhn_rpm import get_header_byte_range + +from server import rhnSQL +from server.importlib import importLib, userAuth, mpmSource, backendOracle, \ + packageImport, errataCache +from server.rhnLib import get_package_path, \ + get_package_path_without_package_name +from server.rhnServer import server_packages + +class BasePackageUpload: + def __init__(self, req): + self.header_prefix = "X-RHN-Upload" + self.error_header_prefix = 'X-RHN-Upload-Error' + self.prefix = 'rhn/repository' + self.is_source = 0 + self.rel_package_path = None + self.package_path = None + self.required_fields = [ + "Package-Name", + "Package-Version", + "Package-Release", + "Package-Arch", + "File-MD5sum", + ] + self.field_data = UserDictCase() + self.org_id = None + + def headerParserHandler(self, req): + """ This whole function is ugly as hell. The Auth field in the header used to be required, but now + it must have either the Auth field or the Auth-Session field. + """ + # Initialize the logging + log_debug(3, "Method", req.method) + + #Header string. This is what the Auth-Session field will look like in the header. + session_header = "%s-%s" % (self.header_prefix, "Auth-Session") + + for f in self.required_fields: + hf = "%s-%s" % (self.header_prefix, f) + if not req.headers_in.has_key(hf): + #If the current field is Auth and Auth-Session field isn't present, something is wrong. + if f == "Auth" and not req.headers_in.has_key(session_header): + log_debug(4, "Required field %s missing" % f) + raise rhnFault(500, f) + + #The current field is Auth and the Auth-Session field is present, so everything is good. + elif f == "Auth" and req.headers_in.has_key(session_header): + self.field_data["Auth-Session"] = req.headers_in[session_header] + continue + + #The current field being looked for isn't the Auth field and it's missing, so something is wrong. + else: + log_debug(4, "Required field %s missing" % f) + raise rhnFault(500, f) + + if not (f == "Auth" and not req.headers_in.has_key(hf)): + self.field_data[f] = req.headers_in[hf] + else: + if req.headers_in.has_key(session_header): + self.field_data[f] = req.headers_in[hf] + + self.package_name = self.field_data["Package-Name"] + self.package_version = self.field_data["Package-Version"] + self.package_release = self.field_data["Package-Release"] + self.package_arch = self.field_data["Package-Arch"] + self.file_md5sum = self.field_data["File-MD5sum"] + #4/18/05 wregglej. if 1051 is in the header's keys, then it's a nosrc package. + self.is_source = (self.package_arch == 'src' or self.package_arch == 'nosrc') + return apache.OK + + def handler(self, req): + log_debug(3, "Method", req.method) + return apache.OK + def cleanupHandler(self, req): + return apache.OK + + def logHandler(self, req): + return apache.OK diff --git a/backend/server/rhnPackage.py b/backend/server/rhnPackage.py index c5f97b0..0b7ac08 100644 --- a/backend/server/rhnPackage.py +++ b/backend/server/rhnPackage.py @@ -432,6 +432,22 @@ def get_channels_for_package(pkg): return [] return map(lambda c: c['label'], ret)
+def get_package_for_md5sum(org_id, md5sum): + statement = """ + select + p.id + from + rhnPackage p + where p.org_id = :org_id + and p.md5sum = :md5sum + """ + h = rhnSQL.prepare(statement) + h.execute(org_id=org_id, md5sum=md5sum) + ret = h.fetchone_dict() + if not ret: + return None + return ret['id'] +
def _none2emptyString(foo): if foo is None: diff --git a/backend/server/rhnPackageUpload.py b/backend/server/rhnPackageUpload.py index 5cdcd95..46b57a7 100644 --- a/backend/server/rhnPackageUpload.py +++ b/backend/server/rhnPackageUpload.py @@ -17,7 +17,6 @@
import os import tempfile -from mod_python import apache
from common import CFG, log_debug, rhnFault, rhn_mpm, rhnLib, UserDictCase from common.rhn_rpm import get_header_byte_range @@ -29,76 +28,6 @@ from server.rhnLib import get_package_path, \ get_package_path_without_package_name from server.rhnServer import server_packages
-class BasePackageUpload: - def __init__(self, req): - self.header_prefix = "X-RHN-Upload" - self.error_header_prefix = 'X-RHN-Upload-Error' - self.prefix = 'rhn/repository' - self.is_source = 0 - self.rel_package_path = None - self.package_path = None - self.required_fields = [ - "Package-Name", - "Package-Version", - "Package-Release", - "Package-Arch", - "File-MD5sum", - ] - self.field_data = UserDictCase() - self.org_id = None - - def headerParserHandler(self, req): - """ This whole function is ugly as hell. The Auth field in the header used to be required, but now - it must have either the Auth field or the Auth-Session field. - """ - # Initialize the logging - log_debug(3, "Method", req.method) - - #Header string. This is what the Auth-Session field will look like in the header. - session_header = "%s-%s" % (self.header_prefix, "Auth-Session") - - for f in self.required_fields: - hf = "%s-%s" % (self.header_prefix, f) - if not req.headers_in.has_key(hf): - #If the current field is Auth and Auth-Session field isn't present, something is wrong. - if f == "Auth" and not req.headers_in.has_key(session_header): - log_debug(4, "Required field %s missing" % f) - raise rhnFault(500, f) - - #The current field is Auth and the Auth-Session field is present, so everything is good. - elif f == "Auth" and req.headers_in.has_key(session_header): - self.field_data["Auth-Session"] = req.headers_in[session_header] - continue - - #The current field being looked for isn't the Auth field and it's missing, so something is wrong. - else: - log_debug(4, "Required field %s missing" % f) - raise rhnFault(500, f) - - if not (f == "Auth" and not req.headers_in.has_key(hf)): - self.field_data[f] = req.headers_in[hf] - else: - if req.headers_in.has_key(session_header): - self.field_data[f] = req.headers_in[hf] - - self.package_name = self.field_data["Package-Name"] - self.package_version = self.field_data["Package-Version"] - self.package_release = self.field_data["Package-Release"] - self.package_arch = self.field_data["Package-Arch"] - self.file_md5sum = self.field_data["File-MD5sum"] - #4/18/05 wregglej. if 1051 is in the header's keys, then it's a nosrc package. - self.is_source = (self.package_arch == 'src' or self.package_arch == 'nosrc') - return apache.OK - - def handler(self, req): - log_debug(3, "Method", req.method) - return apache.OK - def cleanupHandler(self, req): - return apache.OK - - def logHandler(self, req): - return apache.OK -
def source_match(v1, v2): """ returns true if both parameters are true, false otherwise """ diff --git a/backend/upload_server/handlers/package_push/package_push.py b/backend/upload_server/handlers/package_push/package_push.py index fd19a3e..2115ada 100644 --- a/backend/upload_server/handlers/package_push/package_push.py +++ b/backend/upload_server/handlers/package_push/package_push.py @@ -26,11 +26,11 @@ from mod_python import apache from rhn import rpclib
from common import CFG, log_debug, log_error, rhnFault, rhnFlags -from server import rhnPackageUpload, rhnSQL +from server import rhnPackageUpload, rhnSQL, basePackageUpload
-class PackagePush(rhnPackageUpload.BasePackageUpload): +class PackagePush(basePackageUpload.BasePackageUpload): def __init__(self, req): - rhnPackageUpload.BasePackageUpload.__init__(self, req) + basePackageUpload.BasePackageUpload.__init__(self, req) self.required_fields.extend([ 'Auth', 'Force', @@ -43,7 +43,7 @@ class PackagePush(rhnPackageUpload.BasePackageUpload): self.force = None
def headerParserHandler(self, req): - ret = rhnPackageUpload.BasePackageUpload.headerParserHandler(self, req) + ret = basePackageUpload.BasePackageUpload.headerParserHandler(self, req) # Optional headers maps = [['Null-Org', 'null_org'], ['Packaging', 'packaging']] for hn, sn in maps: @@ -108,7 +108,7 @@ class PackagePush(rhnPackageUpload.BasePackageUpload):
def handler(self, req): - ret = rhnPackageUpload.BasePackageUpload.handler(self, req) + ret = basePackageUpload.BasePackageUpload.handler(self, req) if ret != apache.OK: return ret
diff --git a/backend/upload_server/handlers/package_upload/package_upload.py b/backend/upload_server/handlers/package_upload/package_upload.py index c3689cb..b087ca9 100644 --- a/backend/upload_server/handlers/package_upload/package_upload.py +++ b/backend/upload_server/handlers/package_upload/package_upload.py @@ -23,12 +23,12 @@ import os from mod_python import apache
from common import CFG, log_debug, log_error, rhnFault, rhnFlags -from server import rhnPackageUpload +from server import rhnPackageUpload, basePackageUpload from server.rhnLib import get_package_path
-class PackageUpload(rhnPackageUpload.BasePackageUpload): +class PackageUpload(basePackageUpload.BasePackageUpload): def headerParserHandler(self, req): - ret = rhnPackageUpload.BasePackageUpload.headerParserHandler(self, req) + ret = basePackageUpload.BasePackageUpload.headerParserHandler(self, req) if ret != apache.OK: return ret
@@ -60,7 +60,7 @@ class PackageUpload(rhnPackageUpload.BasePackageUpload): return apache.OK
def handler(self, req): - ret = rhnPackageUpload.BasePackageUpload.handler(self, req) + ret = basePackageUpload.BasePackageUpload.handler(self, req) if ret != apache.OK: return ret
spacewalk-commits@lists.fedorahosted.org