• Откуда: Вильнюс (Литва)
  • Зарегистрирован: 19-05-2005
  • Сообщений: 1,103
  • User Karma: 11

Тема: Pulp управление репозиториями и зеркалами

Понадобилось поднять внутреннее зеркало и свой репозиторий в который поместим пакеты которые пользуем для кластеров...

ссылочки которые я почитал и мне понравились

https://docs.pulpproject.org/user-guide … /f23-.html
https://gist.github.com/omaciel/e52c78d48c520101368a
https://docs.pulpproject.org/dev-guide/ … index.html
https://gist.github.com/nextrevision/8f … b58da2d241

процесс установки досточно хорошо описан в документации, тут я оставлю свои комментарии и примеры. Pulp входит в Foreman и Satellite, легко дорабатывается. Имеет полноценный CLI клиент и REST API

1. Install epel or pulp repo

https://repos.fedorapeople.org/repos/pu … -pulp.repo

2. Mongo без авторизации и надо перевесить на localhost

3. QPid  без авторизации и надо перевесить на localhost

4. pulp-admin

pulp-admin auth config file.

# cat /root/.pulp/admin.conf
[auth]
username=admin
password=admin

не красивый способ

pulp-admin -uadmin -padmin

с -vvv на много больше информации. Так же помогает понять как работает API

# pulp-admin -vvv rpm repo uploads rpm --repo-id=lrv-elk-centos-7-x86_64 --file=/home/centos/journalbeat-5.5.0-2.x86_64.rpm
+----------------------------------------------------------------------+
                              Unit Upload
+----------------------------------------------------------------------+
2017-07-25 10:26:34,956 - DEBUG - sending GET request to /pulp/api/v2/repositories/lrv-elk-centos-7-x86_64/
2017-07-25 10:26:35,062 - INFO - GET request to /pulp/api/v2/repositories/lrv-elk-centos-7-x86_64/ with parameters None
2017-07-25 10:26:35,063 - INFO - Response status : 200
2017-07-25 10:26:35,063 - INFO - Response body :
 {
  "scratchpad": {},
  "display_name": "ELK for CentOS 7 Repo",
  "description": "ELK for CentOS 7 Repo",
  "last_unit_added": "2017-07-24T11:12:35Z",
  "notes": {
    "_repo-type": "rpm-repo"
  },
  "last_unit_removed": null,
  "content_unit_counts": {
    "rpm": 1
  },
  "_ns": "repos",
  "_id": {
    "$oid": "5975d5bb0bdd4d035ae8c385"
  },
  "id": "lrv-elk-centos-7-x86_64",
  "_href": "/pulp/api/v2/repositories/lrv-elk-centos-7-x86_64/"
}
Extracting necessary metadata for each request...
[==================================================] 100%
Analyzing: journalbeat-5.5.0-2.x86_64.rpm
... completed
Creating upload requests on the server...
[==================================================] 100%
Initializing: journalbeat-5.5.0-2.x86_64.rpm
2017-07-25 10:26:35,063 - DEBUG - sending POST request to /pulp/api/v2/content/uploads/
2017-07-25 10:26:35,164 - INFO - POST request to /pulp/api/v2/content/uploads/ with parameters None
2017-07-25 10:26:35,164 - INFO - Response status : 201
2017-07-25 10:26:35,165 - INFO - Response body :
 {
  "upload_id": "febb843f-619d-4435-a158-29ac721608c6",
  "_href": "/pulp/api/v2/content/uploads/febb843f-619d-4435-a158-29ac721608c6/"
}
... completed
Starting upload of selected units. If this process is stopped through ctrl+c,
the uploads will be paused and may be resumed later using the resume command or
canceled entirely using the cancel command.
Uploading: journalbeat-5.5.0-2.x86_64.rpm
2017-07-25 10:26:35,169 - DEBUG - sending PUT request to /pulp/api/v2/content/uploads/febb843f-619d-4435-a158-29ac721608c6/0/
2017-07-25 10:26:35,276 - INFO - PUT request to /pulp/api/v2/content/uploads/febb843f-619d-4435-a158-29ac721608c6/0/
2017-07-25 10:26:35,276 - INFO - Response status : 200
2017-07-25 10:26:35,276 - INFO - Response body :
 null
[============                                      ] 24%
1048576/4347520 bytes
2017-07-25 10:26:35,277 - DEBUG - sending PUT request to /pulp/api/v2/content/uploads/febb843f-619d-4435-a158-29ac721608c6/1048576/
2017-07-25 10:26:35,386 - INFO - PUT request to /pulp/api/v2/content/uploads/febb843f-619d-4435-a158-29ac721608c6/1048576/
2017-07-25 10:26:35,386 - INFO - Response status : 200
2017-07-25 10:26:35,386 - INFO - Response body :
[========================                          ] 48%
2097152/4347520 bytes
2017-07-25 10:26:35,387 - DEBUG - sending PUT request to /pulp/api/v2/content/uploads/febb843f-619d-4435-a158-29ac721608c6/2097152/
2017-07-25 10:26:35,497 - INFO - PUT request to /pulp/api/v2/content/uploads/febb843f-619d-4435-a158-29ac721608c6/2097152/
2017-07-25 10:26:35,498 - INFO - Response status : 200
2017-07-25 10:26:35,498 - INFO - Response body :
[====================================              ] 72%
3145728/4347520 bytes
2017-07-25 10:26:35,499 - DEBUG - sending PUT request to /pulp/api/v2/content/uploads/febb843f-619d-4435-a158-29ac721608c6/3145728/
2017-07-25 10:26:35,607 - INFO - PUT request to /pulp/api/v2/content/uploads/febb843f-619d-4435-a158-29ac721608c6/3145728/
2017-07-25 10:26:35,607 - INFO - Response status : 200
2017-07-25 10:26:35,607 - INFO - Response body :
[================================================  ] 96%
4194304/4347520 bytes
2017-07-25 10:26:35,608 - DEBUG - sending PUT request to /pulp/api/v2/content/uploads/febb843f-619d-4435-a158-29ac721608c6/4194304/
2017-07-25 10:26:35,751 - INFO - PUT request to /pulp/api/v2/content/uploads/febb843f-619d-4435-a158-29ac721608c6/4194304/
2017-07-25 10:26:35,751 - INFO - Response status : 200
2017-07-25 10:26:35,751 - INFO - Response body :
[==================================================] 100%
4347520/4347520 bytes
... completed

5. Обновление базы, иногда надо

# sudo -u apache pulp-manage-db

6. Перегружаем все компоненты

# for s in {pulp_celerybeat,pulp_resource_manager,pulp_workers,httpd}; do sudo systemctl restart $s; done;

Repo management

http://dnaeon.github.io/managing-reposi … with-pulp/

Create

# pulp-admin rpm repo create --repo-id=centos-7-x86_64-base --description 'CentOS 7 Base Repo' --display-name 'CentOS 7 Base Repo' --feed=http://mirror.centos.org/centos/7/os/x86_64/
# pulp-admin rpm repo create --repo-id=centos-7-x86_64-updates --description 'CentOS 7 Updates Repo' --display-name 'CentOS 7 Updates Repo' --feed=http://mirror.centos.org/centos/7/updates/x86_64/

Schedule updates

Schedule time format https://en.wikipedia.org/wiki/ISO_8601

# pulp-admin rpm repo sync schedules create --schedule 2017-07-20T01:11:00Z/P1DT --repo-id centos-7-x86_64-base
# pulp-admin rpm repo sync schedules create --schedule 2017-07-20T01:41:00Z/P1DT --repo-id centos-7-x86_64-updates

Run sync ASAP

# pulp-admin rpm repo sync run --repo-id centos-7-x86_64-base

or via API

# curl -k -s -u admin:admin -H "Accept:application/json,version=2" -H "Content-Type:application/json" -X POST -d '{"id": "centos-7-x86_64-updates"}' https://localhost/pulp/api/v2/repositories/centos-7-x86_64-updates/actions/sync/ | jq
{
  "spawned_tasks": [
    {
      "_href": "/pulp/api/v2/tasks/b7fb680d-7363-489a-891e-af8c1d46e8d7/",
      "task_id": "b7fb680d-7363-489a-891e-af8c1d46e8d7"
    }
  ],
  "result": null,
  "error": null
}

Show sync status

# pulp-admin rpm repo sync status --repo-id centos-7-x86_64-base

via API

# curl -k -s -u admin:admin https://localhost/pulp/api/v2/tasks/b7fb680d-7363-489a-891e-af8c1d46e8d7/ | jq                                   {
  "exception": null,
  "task_type": "pulp.server.managers.repo.sync.sync",
  "_href": "/pulp/api/v2/tasks/b7fb680d-7363-489a-891e-af8c1d46e8d7/",
  "task_id": "b7fb680d-7363-489a-891e-af8c1d46e8d7",
  "tags": [
    "pulp:repository:centos-7-x86_64-base",
    "pulp:action:sync"
  ],
  "finish_time": "2017-07-24T10:39:06Z",
  "_ns": "task_status",
  "start_time": "2017-07-24T10:39:00Z",
  "traceback": null,
  "spawned_tasks": [
    {
      "_href": "/pulp/api/v2/tasks/95691763-5ff6-44fa-a14a-815bbfebed88/",
      "task_id": "95691763-5ff6-44fa-a14a-815bbfebed88"
    }
  ],
  "progress_report": {
    "yum_importer": {
      "content": {
        "items_total": 0,
        "state": "FINISHED",
        "error_details": [],
        "details": {
          "rpm_total": 0,
          "rpm_done": 0,
          "drpm_total": 0,
          "drpm_done": 0
        },
        "size_total": 0,
        "size_left": 0,
        "items_left": 0
      },
      "comps": {
        "state": "FINISHED"
      },
      "purge_duplicates": {
        "state": "FINISHED"
      },
      "distribution": {
        "items_total": 0,
        "state": "FINISHED",
        "error_details": [],
        "items_left": 0
      },
      "errata": {
        "state": "FINISHED"
      },
      "metadata": {
        "state": "FINISHED"
      }
    }
  },
  "queue": "[email protected]",
  "state": "finished",
  "worker_name": "[email protected]",
  "result": {
    "result": "success",
    "importer_id": "yum_importer",
    "exception": null,
    "repo_id": "centos-7-x86_64-base",
    "traceback": null,
    "started": "2017-07-24T10:39:00Z",
    "_ns": "repo_sync_results",
    "completed": "2017-07-24T10:39:05Z",
    "importer_type_id": "yum_importer",
    "error_message": null,
    "summary": {
      "content": {
        "state": "FINISHED"
      },
      "comps": {
        "state": "FINISHED"
      },
      "purge_duplicates": {
        "state": "FINISHED"
      },
      "distribution": {
        "state": "FINISHED"
      },
      "errata": {
        "state": "FINISHED"
      },
      "metadata": {
        "state": "FINISHED"
      }
    },
    "added_count": 0,
    "removed_count": 0,
    "updated_count": 0,
    "id": "5975ce490bdd4d05dd0426b6",
    "details": {
      "content": {
        "size_total": 0,
        "items_left": 0,
        "items_total": 0,
        "state": "FINISHED",
        "size_left": 0,
        "details": {
          "rpm_total": 0,
          "rpm_done": 0,
          "drpm_total": 0,
          "drpm_done": 0
        },
        "error_details": []
      },
      "comps": {
        "state": "FINISHED"
      },
      "purge_duplicates": {
        "state": "FINISHED"
      },
      "distribution": {
        "items_total": 0,
        "state": "FINISHED",
        "error_details": [],
        "items_left": 0
      },
      "errata": {
        "state": "FINISHED"
      },
      "metadata": {
        "state": "FINISHED"
      }
    }
  },
  "error": null,
  "_id": {
    "$oid": "5975ce445f5d7b090840d693"
  },
  "id": "5975ce445f5d7b090840d693"
}

Custom (own) rpm repo

свой репозиторий

# pulp-admin rpm repo create --repo-id=lrv-elk-centos-7-x86_64 --description 'ELK for CentOS 7 Repo' --display-name 'ELK for CentOS 7 Repo' --relative-url=lrv-elk/centos/7/x86_64/

заливаем rpm

# pulp-admin rpm repo uploads rpm --repo-id=lrv-elk-centos-7-x86_64 --file=/home/centos/journalbeat-5.5.0-1.x86_64.rpm

выставляем

# pulp-admin rpm repo publish run --repo-id=lrv-elk-centos-7-x86_64

yum.repo пример
Есть плагин который генерирует список зеркал и т.п.

[lrv-elk]
# cat lrv-elk.repo
name=ELK for CentOS 7 Repo
baseurl=https://1.1.1.1/pulp/repos/lrv-elk/centos/7/$basearch
enabled=1
gpgcheck=0
sslverify=0

Добавлено: 26-07-2018 16:52:18

Дополнение, прикрутил ко всему этому deb поддержду, все работает по мануалу https://github.com/pulp/pulp_deb. Ручками только надо было добавить допаись GPG пакетов и репозитория.

[root@pulp ~]# cat /etc/pulp/server/plugins.conf.d/deb_distributor.json 
{
  "gpg_cmd": "/usr/local/bin/sign.sh",
  "gpg_key_id": "0000000"
}
[root@pulp ~]# cat /usr/local/bin/sign.sh
#!/bin/bash -e
KEYID=${GPG_KEY_ID:-00000000}
logger -t "GPG-Sign" "(${1}) ($KEYID)"
RDir=`dirname "${1}"`
RFile=`basename "${1}"`
rm -fv ${RDir}/In${RFile} ${RDir}/${RFile}.gpg | tee >(logger -t "GPG-Sign")
/usr/bin/gpg2 \
    --homedir /var/lib/pulp/gpg-home \
    --no-tty \
    --batch \
    --digest-algo SHA256 \
    --passphrase "XXX" \
    --default-key $KEYID \
    --clearsign \
    --output ${RDir}/In${RFile} ${1} 2>&1 | tee >(logger -t "GPG-Sign")
/usr/bin/gpg2 \
    --homedir /var/lib/pulp/gpg-home \
    --no-tty \
    --batch \
    --digest-algo SHA256 \
    --passphrase "XXX" \
    --default-key $KEYID \
    -abs \
    --output ${1}.gpg ${1} 2>&1 | tee >(logger -t "GPG-Sign")

Ну и выставляем на зеркало и по пути подписываем пакеты

[root@pulp ~]# cat /srv/pulpsync/sync.sh 
#!/bin/bash
NEWFILES=`find /var/www/pub/ -mmin -9 -type f | wc -l`
DO_RePUBLUSH=0
if [ $NEWFILES -ne 0 ]; then
  echo "($NEWFILES files)" | tee >(logger -t "pulp-sync")
# SIGN
  echo "start singning" | tee >(logger -t "pulp-sync")
  NEW_DEB_FILES=`find /var/www/pub/deb -type l -name '*.deb'`
  cd /var/www
  for i in $NEW_DEB_FILES
  do
    /usr/bin/dpkg-sig -p --verify $i | grep -q "00000000000000000000"
    if [ $? -ne 0 ]; then
      DO_RePUBLUSH=1
      echo "Unsigned file $i found" | tee >(logger -t "pulp-sync")
      sudo -u apache /usr/bin/dpkg-sig -p -k 0000000 -f /var/lib/pulp/gpg-home/.pass_0000000 --sign builder $i | tee >(logger -t "pulp-sync")
    else
      echo "test signature for $i" | tee >(logger -t "pulp-sync")
    fi
  done
# RePublish
if [ $DO_RePUBLUSH -ne 0 ]; then
  echo "start republish" | tee >(logger -t "pulp-sync")
  /usr/bin/pulp-admin deb repo publish run --repo-id test-debian --force-full | tee >(logger -t "pulp-sync")
else
  echo "no republish needed" | tee >(logger -t "pulp-sync")
fi
# SYNC
  echo "start sync" | tee >(logger -t "pulp-sync")
rsync \
    -PavLc \
    --delete-delay \
    -e "ssh -i /srv/pulpsync/.ssh/id_rsa -p2222" \
    /var/www/pub/deb/https/repos/test-debian pulpsync@mirror:/var/www/repo.domain.com/deb/ | tee >(logger -t "pulp-sync")
rsync \
    -PavLc \
    --delete-delay \
    -e "ssh -i /srv/pulpsync/.ssh/id_rsa -p2222" \
    /var/www/pub/gpg/ pulpsync@mirror:/var/www/repo.domain.com/gpg/ | tee >(logger -t "pulp-sync")
ssh -i /srv/pulpsync/.ssh/id_rsa -p2222 pulpsync@mirror "find /var/www/repo.domain.com/ -type f | xargs chmod 644"
ssh -i /srv/pulpsync/.ssh/id_rsa -p2222 pulpsync@mirror "find /var/www/repo.domain.com/ -type d | xargs chmod 755"
else
  echo "no new files found for sync" | tee >(logger -t "pulp-sync")
fi
[root@pulp ~]# cat /etc/cron.d/pulp_sync 
*/10 * * * * root /srv/pulpsync/sync.sh

/usr/bin/dpkg-sig по сути copy-past с дебиана, дополнительно надо было поставить пару perl пакетов, они идут всесте с cpan, так что просто 

yum install cpan

и все