Skip to content
Snippets Groups Projects
Commit 9daa5bc2 authored by root's avatar root Committed by root
Browse files

startup

parent 79ae8929
Branches develop
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 3119 additions and 0 deletions
Dockerfile 0 → 100644
+ 46
− 0
View file @ 9daa5bc2
FROM centos:7
# Install FTS
RUN yum install -y epel-release.noarch
RUN curl https://fts-repo.web.cern.ch/fts-repo/fts3-prod-el7.repo > /etc/yum.repos.d/fts3-prod-el7.repo
RUN curl https://dmc-repo.web.cern.ch/dmc-repo/dmc-el7.repo > /etc/yum.repos.d/dmc-el7.repo
RUN yum upgrade -y
RUN yum install -y gfal2-plugin-*
RUN yum install -y fts-server fts-client fts-rest fts-monitoring fts-mysql fts-msg mysql multitail
RUN yum clean all
# Setup FTS security
COPY assets/fts/certs/hostcert_fts.pem /etc/grid-security/hostcert.pem
COPY assets/fts/certs/hostcert_fts.key.pem /etc/grid-security/hostkey.pem
RUN chmod 400 /etc/grid-security/hostkey.pem
# Database configuration for FTS server
COPY assets/fts/fts3config /etc/fts3/fts3config
COPY assets/fts/mysql/fts-schema-6.0.0.sql /usr/share/fts-mysql/fts-schema-6.0.0.sql
# Configuration for FTSREST and FTSMON
COPY assets/fts/fts3rest.conf /etc/httpd/conf.d/fts3rest.conf
RUN echo "" > /etc/httpd/conf.d/ssl.conf &&\
echo "" > /etc/httpd/conf.d/autoindex.conf &&\
echo "" > /etc/httpd/conf.d/userdir.conf &&\
echo "" > /etc/httpd/conf.d/welcome.conf &&\
echo "" > /etc/httpd/conf.d/zgridsite.conf &&\
echo "ServerName vm-131-154-97-104.cloud.cnaf.infn.it:443" >> /etc/httpd/conf/httpd.conf
# echo "LoadModule mpm_event_module modules/mod_mpm_event.so" >> /etc/httpd/conf/httpd.conf
# Entrypoint waiting script for MySQL
COPY assets/fts/wait-for-it.sh /usr/local/bin/wait-for-it.sh
RUN chmod +x /usr/local/bin/wait-for-it.sh
# Shortcut for logfiles
COPY assets/fts/logshow /usr/local/bin/logshow
RUN chmod +x /usr/local/bin/logshow
RUN touch /var/log/fts3/fts3server.log
RUN chown -R fts3:fts3 /var/log/fts3/fts3server.log
RUN touch /var/log/fts3rest/fts3rest.log
RUN chown -R fts3:fts3 /var/log/fts3rest
# Startup
EXPOSE 8446 8449
ADD assets/fts/docker-entrypoint.sh /
ENTRYPOINT ["/docker-entrypoint.sh"]
assets/fts/certs/hostcert_fts.key.pem 0 → 100644
+ 27
− 0
View file @ 9daa5bc2
-----BEGIN RSA PRIVATE KEY-----
MIIEpQIBAAKCAQEA1N4nyDnNgCxUbnxowHm8D9frz/WxIYMY/OLv8VXXsJyQ3oWM
UAuOavgVdiUVi8dgwrrDHLBaU56iQwx9Mj5z4L0wDOJPibbsCLsRF647vkKLSO/d
IJm5BC8/cCBznnkX5vAwAO74fhfbcaMpNRbbRk48FXssd4GfouzBvQlybAcsjUis
Oc0q+5FYPf6HYmhQGP/vqncMFyccCUPV4yxCO+jtehRsGC7guTtB+QDXQdOleaEO
qgrK+m/ga2X++Dmh49+xqQj+FH3d/Ia+WyAIPxXZ54DjNFwl3RGN8Y8REp2bruAt
QfLz0bD6/ZmOAFbFL0DDkr8/sa/vtVGRJ77YaQIDAQABAoIBAQCN/fKDeSs/SFK5
sHsfiRSNbPZ7reQN+qj8Z1SuoxhRUt6/0E2cR1kkuX93Y51PtBszSelEhc564OJT
30zT6xzY2Nnofj2YG1CvMjlglnR7oV/Ps1DJokDsLagfX0ZAe34aKjWbKczQL5HN
54xhQXHbH+ysV5F91hVqVIuXUWnl7FqlMXJ7iEd2wpMl6FDNqZUaKc7KEeJKPFHN
HqKz7uX7I9TSnMIv4SK7IUPBc0qP1s0W257SQ40ABktHilbWxFIhf5xr/HtnrvO6
lpn9TEC4XwGURAZIcvSDcnkakPna3uOuJhimaoyuz2eARo/4ZTfO9m0EMT3DAe4l
S0QGqk5BAoGBAP2qWmnuowajPg3VtMW9cuxEEKdPY/4Nt6R3VRhp5Ax1cAnRVuSV
BqO/HXyD1TUoprakioYtclvLIPj80tlFRyyAPPg7CF9CBlrZhRF2nmJiZ3iDKagI
DddHU2Dpw30MIR6WF4GggTOpGvAUSHP+I5qzZ+cP+DXSBLOjKUJQho59AoGBANbT
rmJC8zgA2TeqWMFaCO7IsLw2D4oE2juqqmnh1knQcLDumsl4aNl4fQT9qs00FOUC
Dssd4O+G5PYzoXgBEh9XreLkc66n3kzCxHcco/O+L1FT0Uc7ziYqI6sjhFf3oZPD
3Jx47Pc4X5GH0Hq7Z+RZ/Ik/Na6E2y05hljm1HldAoGBALAjUSLLjLI4OgebRCom
cZ1zaejzbKhlWAZAwnrwVGT+SuzWSrLwdCrRsyTZ2ftOneym9nVTK1xoAR5DWBl8
TH6vEScAvsB+dTEyZ38H0dTZJaX91bgS16GLqm7aA19aQ8kcSXg+Pljj7IuWWfUJ
HCn4t4YvhZfBnHgsLKMajUR1AoGBANCihJmpq8yYBl2pcslVR5egEQS4/B5u2ANa
nGZ4jkThypALKwi/iBSsj75pHZhtHF3MwdFCE1KND5CMVCTX5WHLJXVVqYPWRG8I
OQX1KsVtz0aMwJMN4gfupPVwaPeF58Y6UeclCf6PIUGVRUPs/dkoC7q/SMNVBAfH
9lIg2/mBAoGAF8Uo0fddAOcUvto1yH5a3pWuM/F4eHwmV/C7cLLy3R3ry6wjFuS0
sv0LRObwNU8Is1nYdzBcngDJkFNkkYGUowxMn13V85TYCfXX93IwwKiqQhRTYLvQ
RCA6LybZgH0J7djWzYYGvPpeq0RLwEd7GNjGp0Ujp+VM4Bzt3n7M3ro=
-----END RSA PRIVATE KEY-----
assets/fts/certs/hostcert_fts.pem 0 → 100644
+ 85
− 0
View file @ 9daa5bc2
Certificate:
Data:
Version: 3 (0x2)
Serial Number: 108 (0x6c)
Signature Algorithm: sha1WithRSAEncryption
Issuer: C=IT, O=IGI, CN=Test CA
Validity
Not Before: Nov 20 16:08:33 2014 GMT
Not After : Nov 17 16:08:33 2024 GMT
Subject: C=IT, O=IGI, CN=*.cloud.cnaf.infn.it
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
RSA Public Key: (2048 bit)
Modulus (2048 bit):
00:d4:de:27:c8:39:cd:80:2c:54:6e:7c:68:c0:79:
bc:0f:d7:eb:cf:f5:b1:21:83:18:fc:e2:ef:f1:55:
d7:b0:9c:90:de:85:8c:50:0b:8e:6a:f8:15:76:25:
15:8b:c7:60:c2:ba:c3:1c:b0:5a:53:9e:a2:43:0c:
7d:32:3e:73:e0:bd:30:0c:e2:4f:89:b6:ec:08:bb:
11:17:ae:3b:be:42:8b:48:ef:dd:20:99:b9:04:2f:
3f:70:20:73:9e:79:17:e6:f0:30:00:ee:f8:7e:17:
db:71:a3:29:35:16:db:46:4e:3c:15:7b:2c:77:81:
9f:a2:ec:c1:bd:09:72:6c:07:2c:8d:48:ac:39:cd:
2a:fb:91:58:3d:fe:87:62:68:50:18:ff:ef:aa:77:
0c:17:27:1c:09:43:d5:e3:2c:42:3b:e8:ed:7a:14:
6c:18:2e:e0:b9:3b:41:f9:00:d7:41:d3:a5:79:a1:
0e:aa:0a:ca:fa:6f:e0:6b:65:fe:f8:39:a1:e3:df:
b1:a9:08:fe:14:7d:dd:fc:86:be:5b:20:08:3f:15:
d9:e7:80:e3:34:5c:25:dd:11:8d:f1:8f:11:12:9d:
9b:ae:e0:2d:41:f2:f3:d1:b0:fa:fd:99:8e:00:56:
c5:2f:40:c3:92:bf:3f:b1:af:ef:b5:51:91:27:be:
d8:69
Exponent: 65537 (0x10001)
X509v3 extensions:
X509v3 Basic Constraints: critical
CA:FALSE
X509v3 Subject Key Identifier:
D7:B5:B8:F2:25:B5:27:25:72:64:E6:91:09:D2:D4:52:ED:E5:E1:89
X509v3 Key Usage: critical
Digital Signature, Non Repudiation, Key Encipherment
X509v3 Extended Key Usage:
TLS Web Server Authentication, TLS Web Client Authentication, Microsoft Server Gated Crypto, Netscape Server Gated Crypto, E-mail Protection
X509v3 Authority Key Identifier:
keyid:91:77:36:7B:2E:B4:69:F3:27:EA:B7:F6:08:8B:4A:23:A2:11:49:C6
X509v3 Subject Alternative Name:
DNS:*.cloud.cnaf.infn.it
Signature Algorithm: sha1WithRSAEncryption
68:04:cd:18:ca:cf:bf:a3:a8:aa:d8:e7:0b:1c:c9:8b:12:a2:
1d:2e:4a:e7:d2:bd:70:0c:23:24:07:cc:9c:be:b8:8d:2d:74:
b7:d5:48:93:c7:c8:c1:64:b4:28:70:64:89:90:8b:76:94:63:
b6:00:bc:1c:9c:50:8e:d8:d4:43:97:97:38:e6:9c:c6:bb:65:
34:9f:5e:0c:b5:a7:19:6d:de:8f:41:a5:e0:74:47:bf:71:6f:
84:d7:4a:72:4c:bb:00:ba:e6:0b:cd:8b:d9:77:ee:b0:81:7a:
92:f1:3c:48:2d:42:d5:be:7e:71:64:95:5d:60:6e:76:30:f2:
bb:64:ff:3b:65:f0:17:a9:6d:d3:e6:1c:d6:b4:39:84:03:88:
c8:02:9e:ae:48:0e:c5:dd:3c:ed:89:46:10:da:86:57:1b:fb:
c3:e9:20:b8:87:77:31:56:7c:16:ec:52:1d:bb:50:71:51:1b:
0e:85:81:a0:37:ad:7d:18:63:32:7d:84:c9:5d:c9:73:ea:40:
20:08:45:53:4e:b5:26:df:9e:99:61:5e:49:ec:2f:44:83:15:
14:fe:38:03:9c:f8:5a:6a:6c:2b:d7:d1:d9:b6:6c:60:51:04:
af:4a:c8:c5:02:c5:7d:76:ec:c3:a2:e7:3e:01:79:b0:38:8a:
ab:8d:69:22
-----BEGIN CERTIFICATE-----
MIIDpTCCAo2gAwIBAgIBbDANBgkqhkiG9w0BAQUFADAtMQswCQYDVQQGEwJJVDEM
MAoGA1UECgwDSUdJMRAwDgYDVQQDDAdUZXN0IENBMB4XDTE0MTEyMDE2MDgzM1oX
DTI0MTExNzE2MDgzM1owOjELMAkGA1UEBhMCSVQxDDAKBgNVBAoMA0lHSTEdMBsG
A1UEAwwUKi5jbG91ZC5jbmFmLmluZm4uaXQwggEiMA0GCSqGSIb3DQEBAQUAA4IB
DwAwggEKAoIBAQDU3ifIOc2ALFRufGjAebwP1+vP9bEhgxj84u/xVdewnJDehYxQ
C45q+BV2JRWLx2DCusMcsFpTnqJDDH0yPnPgvTAM4k+JtuwIuxEXrju+QotI790g
mbkELz9wIHOeeRfm8DAA7vh+F9txoyk1FttGTjwVeyx3gZ+i7MG9CXJsByyNSKw5
zSr7kVg9/odiaFAY/++qdwwXJxwJQ9XjLEI76O16FGwYLuC5O0H5ANdB06V5oQ6q
Csr6b+BrZf74OaHj37GpCP4Ufd38hr5bIAg/FdnngOM0XCXdEY3xjxESnZuu4C1B
8vPRsPr9mY4AVsUvQMOSvz+xr++1UZEnvthpAgMBAAGjgcIwgb8wDAYDVR0TAQH/
BAIwADAdBgNVHQ4EFgQU17W48iW1JyVyZOaRCdLUUu3l4YkwDgYDVR0PAQH/BAQD
AgXgMD4GA1UdJQQ3MDUGCCsGAQUFBwMBBggrBgEFBQcDAgYKKwYBBAGCNwoDAwYJ
YIZIAYb4QgQBBggrBgEFBQcDBDAfBgNVHSMEGDAWgBSRdzZ7LrRp8yfqt/YIi0oj
ohFJxjAfBgNVHREEGDAWghQqLmNsb3VkLmNuYWYuaW5mbi5pdDANBgkqhkiG9w0B
AQUFAAOCAQEAaATNGMrPv6OoqtjnCxzJixKiHS5K59K9cAwjJAfMnL64jS10t9VI
k8fIwWS0KHBkiZCLdpRjtgC8HJxQjtjUQ5eXOOacxrtlNJ9eDLWnGW3ej0Gl4HRH
v3FvhNdKcky7ALrmC82L2XfusIF6kvE8SC1C1b5+cWSVXWBudjDyu2T/O2XwF6lt
0+Yc1rQ5hAOIyAKerkgOxd087YlGENqGVxv7w+kguId3MVZ8FuxSHbtQcVEbDoWB
oDetfRhjMn2EyV3Jc+pAIAhFU061Jt+emWFeSewvRIMVFP44A5z4WmpsK9fR2bZs
YFEEr0rIxQLFfXbsw6LnPgF5sDiKq41pIg==
-----END CERTIFICATE-----
assets/fts/docker-entrypoint.sh 0 → 100755
+ 11
− 0
View file @ 9daa5bc2
#!/bin/bash
# wait for MySQL readiness
/usr/local/bin/wait-for-it.sh -h ftsdb -p 3306 -t 3600
# initialise / upgrade the database
mysql -u root -pfts -h ftsdb fts < /usr/share/fts-mysql/fts-schema-6.0.0.sql
# startup the FTS services
/usr/sbin/fts_server # main FTS server daemonizes
/usr/sbin/httpd -DFOREGROUND # FTS REST frontend & FTSMON
assets/fts/fts3config 0 → 100644
+ 18
− 0
View file @ 9daa5bc2
Alias=fts
SiteName=DOCKER
AuthorizedVO=*
DbType=mysql
DbUserName=fts
DbPassword=fts
DbConnectString=ftsdb/fts
LogLevel=DEBUG
MinRequiredFreeRAM = 16
MaxUrlCopyProcesses = 4
[roles]
Public = all:transfer;all:config;all:datamanagement
assets/fts/fts3rest.conf 0 → 100644
+ 87
− 0
View file @ 9daa5bc2
# Load required modules
<IfModule !wsgi_module>
LoadModule wsgi_module modules/mod_wsgi.so
</IfModule>
<IfModule !ssl_module>
LoadModule ssl_module modules/mod_ssl.so
</IfModule>
<IfModule !gridsite_module>
LoadModule gridsite_module modules/mod_gridsite.so
</IfModule>
<IfModule !version_module>
LoadModule version_module modules/mod_version.so
</IfModule>
WSGISocketPrefix run/wsgi
#SiteName vm-131-154-97-104.cloud.cnaf.infn.it:443
# Enable REST interface in port 8446
Listen 8446
<VirtualHost *:8446>
# SSL configuration
SSLProtocol all -SSLv2 -SSLv3
SSLCipherSuite RC4-SHA:AES128-SHA:HIGH:!aNULL:!MD5:!RC4
SSLHonorCipherOrder on
# Certificates
SSLCertificateFile /etc/grid-security/hostcert.pem
SSLCertificateKeyFile /etc/grid-security/hostkey.pem
# SSLCACertificateFile /etc/grid-security/certificates/5fca1cb1.0
# Require client certificate
SSLVerifyClient optional
SSLVerifyDepth 10
# Disable the session files of libgridsite
GridSiteGridHTTP off
GridSiteAutoPasscode off
# Export environment variables with SSL information
# Needed by FTS3 REST interface
<IfVersion >= 2.4>
SSLOptions +StdEnvVars +ExportCertData +StdEnvVars +LegacyDNStringFormat
</IfVersion>
<IfVersion < 2.4>
SSLOptions +StdEnvVars +ExportCertData +StdEnvVars
</IfVersion>
# Enable SSL in this port
SSLEngine on
# Logs
ErrorLog logs/fts3rest_error_log
LogFormat "%a %l %u %t \"%r\" %>s %b %D"
TransferLog logs/fts3rest_access_log
LogLevel debug
# Send everything to the FTS3 REST interface
WSGIScriptAlias / /usr/libexec/fts3/fts3rest.wsgi
# Encoded slashes must be kept
AllowEncodedSlashes NoDecode
<Location />
<IfVersion >= 2.4>
Require all granted
</IfVersion>
<IfVersion < 2.4>
Order allow,deny
Allow from all
</IfVersion>
</Location>
# For OAuth2 support, the Authentication header needs to be passed
WSGIPassAuthorization On
# We need to avoid the underlying libraries (i.e. Globus) from modifying the status
# of the server process (httpd), so isolate the application
WSGIDaemonProcess fts3rest processes=2 threads=15 maximum-requests=3000 display-name=fts3rest user=fts3 group=fts3
WSGIProcessGroup fts3rest
WSGIApplicationGroup %{GLOBAL}
# Headers
Header Set "X-Frame-Options" "DENY"
</VirtualHost>
assets/fts/logshow 0 → 100644
+ 2
− 0
View file @ 9daa5bc2
#!/bin/bash
multitail -f /var/log/fts3/fts3server.log /var/log/fts3rest/fts3rest.log
assets/fts/mysql/fts-diff-1.0.1.sql 0 → 100644
+ 15
− 0
View file @ 9daa5bc2
--
-- FTS3 Schema 1.0.1
-- [FTS-530] Database upgrade tool
--
-- t_schema_vers modified to allow longer descriptions
--
ALTER TABLE t_schema_vers
DROP COLUMN state;
ALTER TABLE t_schema_vers
ADD COLUMN message TEXT;
INSERT INTO t_schema_vers (major, minor, patch, message)
VALUES (1, 0, 1, 'FTS-530 diff');
assets/fts/mysql/fts-diff-2.0.0.sql 0 → 100644
+ 82
− 0
View file @ 9daa5bc2
--
-- FTS3 Schema 1.1.0
-- [FTS-308] Allow for range settings for the number of actives
-- [FTS-506] Register on the optimizer actual number of actives (Schema change!)
--
-- t_optimize is modified to allow a range of min/max number of actives per link
--
--
-- FTS 308
--
ALTER TABLE t_optimize_active
ADD COLUMN `min_active` INTEGER DEFAULT NULL,
ADD COLUMN `max_active` INTEGER DEFAULT NULL,
DROP COLUMN `message`;
-- Set range for existing fixed links
UPDATE t_optimize_active
SET min_active = active, max_active = active
WHERE fixed = 'on' AND min_active IS NULL AND max_active IS NULL;
--
-- Update t_optimizer_evolution in three steps
-- It is normally faster, and keeps data
--
-- First, create an empty one, identical
CREATE TABLE t_optimizer_evolution_new LIKE t_optimizer_evolution;
-- Modify the schema in the empty table
ALTER TABLE t_optimizer_evolution_new
CHANGE COLUMN `filesize` `success` FLOAT DEFAULT NULL,
DROP COLUMN `agrthroughput`,
DROP COLUMN `buffer`,
DROP COLUMN `nostreams`,
DROP COLUMN `timeout`,
ADD COLUMN `ema` FLOAT DEFAULT NULL,
ADD COLUMN `rationale` TEXT DEFAULT NULL,
ADD COLUMN `diff` INTEGER DEFAULT 0,
ADD COLUMN actual_active INTEGER DEFAULT NULL,
ADD COLUMN queue_size INTEGER DEFAULT NULL,
ADD COLUMN filesize_avg DOUBLE DEFAULT NULL,
ADD COLUMN filesize_stddev DOUBLE DEFAULT NULL;
-- Populate the empty table with data from the old one, adapted to the
-- new schema
INSERT INTO t_optimizer_evolution_new
(datetime, source_se, dest_se,
active, throughput, success,
rationale)
SELECT datetime, source_se, dest_se,
active, throughput, filesize AS success,
'Entry recovered from old database schema'
FROM t_optimizer_evolution
WHERE datetime > (UTC_TIMESTAMP() - INTERVAL 7 DAY);
-- Drop old table, create new one
DROP TABLE t_optimizer_evolution;
RENAME TABLE t_optimizer_evolution_new TO t_optimizer_evolution;
-- Prepare t_optimize_streams
TRUNCATE t_optimize_streams;
ALTER TABLE t_optimize_streams
DROP COLUMN `throughput`,
DROP COLUMN `tested`,
DROP PRIMARY KEY,
ADD PRIMARY KEY (source_se, dest_se);
-- Drop profiling tables
DROP TABLE IF EXISTS t_profiling_info;
DROP TABLE IF EXISTS t_profiling_snapshot;
-- Drop t_turl
DROP TABLE IF EXISTS t_turl;
--
-- Store update history
--
INSERT INTO t_schema_vers (major, minor, patch, message)
VALUES (2, 0, 0, 'FTS-308, FTS-506, FTS-627, FTS-628 diff');
assets/fts/mysql/fts-diff-2.0.1.sql 0 → 100644
+ 11
− 0
View file @ 9daa5bc2
--
-- FTS3 Schema 2.0.1
-- [FTS-597] Drop unused tables
--
DROP TABLE t_credential_vers;
DROP TABLE t_se_acl;
DROP TABLE t_se_pair_acl;
INSERT INTO t_schema_vers (major, minor, patch, message)
VALUES (2, 0, 1, 'FTS-597 diff');
assets/fts/mysql/fts-diff-3.0.0-partial.sql 0 → 100644
+ 290
− 0
View file @ 9daa5bc2
SET default_storage_engine=InnoDB;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
ALTER TABLE t_activity_share_config
CHANGE COLUMN `activity_share` `activity_share` VARCHAR(1024) NOT NULL;
-- Per https://gitlab.cern.ch/fts/fts3/blob/develop/src/db/schema/unused.md
-- These tables are fairly small, so an in-place modification is reasonable
ALTER TABLE t_optimize
DROP COLUMN `file_id`,
DROP COLUMN `timeout`,
DROP COLUMN `buffer`,
DROP COLUMN `filesize`;
ALTER TABLE t_link_config
DROP COLUMN `placeholder1`,
DROP COLUMN `placeholder2`,
DROP COLUMN `placeholder3`,
DROP COLUMN `NO_TX_ACTIVITY_TO`;
-- For better performance on the web overview
ALTER TABLE t_optimizer_evolution
DROP INDEX t_optimizer_source_and_dest,
ADD INDEX idx_optimizer_evolution(`source_se`, `dest_se`, `datetime`);
-- Reduce size of dlg_id
CREATE TABLE t_credential_cache_new (
`dlg_id` CHAR(16) NOT NULL,
`dn` VARCHAR(255),
`cert_request` LONGTEXT,
`priv_key` LONGTEXT,
`voms_attrs` LONGTEXT,
PRIMARY KEY (dlg_id, dn)
)
AS
SELECT dlg_id, dn, cert_request, priv_key, voms_attrs
FROM t_credential_cache;
RENAME TABLE t_credential_cache TO t_credential_cache_old;
RENAME TABLE t_credential_cache_new TO t_credential_cache;
CREATE TABLE t_credential_new (
`dlg_id` CHAR(16) NOT NULL,
`dn` VARCHAR(255),
`proxy` LONGTEXT,
`voms_attrs` LONGTEXT,
`termination_time` TIMESTAMP NOT NULL,
PRIMARY KEY (dlg_id, dn),
INDEX (termination_time)
)
AS
SELECT dlg_id, dn, proxy, voms_attrs, termination_time
FROM t_credential;
RENAME TABLE t_credential TO t_credential_old;
RENAME TABLE t_credential_new TO t_credential;
-- DROP TABLE t_credential_cache_old;
-- DROP TABLE t_credential_old;
-- Unused + FTS-599 + FTS-617
-- These tables can be quite big, so an in-place modification would take
-- a long time.
-- Thus, we update the schema in two steps:
-- * Replicate a modified table with the data cloned
-- * Add indexes
-- t_job
-- t_file has a foreign key on this, so need to do it first.
-- Dropped fields
-- * job_params varchar(255), unused
-- * agent_dn varchar(1024), unused
-- * user_cred varchar(255), pointless
-- * voms_cred longtext, depends on cred_id
-- * storage_class varchar(255), unused
-- * myproxy_server varchar(255), unused
-- * source_token_description varchar(255), unused
-- * fail_nearline char(1), unused
-- * configuration_count (int), unused
-- * finish_time (timestamp), redundant with job_finished
CREATE TABLE t_job_new (
`job_id` CHAR(36) NOT NULL,
`job_state` ENUM(
'STAGING', 'SUBMITTED', 'READY', 'ACTIVE', 'FINISHED', 'FAILED', 'FINISHEDDIRTY', 'CANCELED', 'DELETE'
) NOT NULL, -- Was job_state varchar(32)
`job_type` CHAR(1) DEFAULT NULL, -- Was reuse_job varchar(3)
`cancel_job` CHAR(1) DEFAULT NULL,
`source_se` VARCHAR(255) DEFAULT NULL,
`dest_se` VARCHAR(255) DEFAULT NULL,
`user_dn` VARCHAR(1024) DEFAULT NULL,
`cred_id` CHAR(16) DEFAULT NULL, -- Was cred_id varchar(100), actually 16 hex digits are stored
`vo_name` VARCHAR(50) DEFAULT NULL,
`reason` VARCHAR(2048) DEFAULT NULL,
`submit_time` TIMESTAMP NULL DEFAULT NULL,
`priority` INT(11) DEFAULT '3',
`submit_host` VARCHAR(255) DEFAULT NULL,
`max_time_in_queue` INT(11) DEFAULT NULL,
`space_token` VARCHAR(255) DEFAULT NULL,
`internal_job_params` VARCHAR(255) DEFAULT NULL,
`overwrite_flag` CHAR(1) DEFAULT NULL,
`job_finished` TIMESTAMP NULL DEFAULT NULL,
`source_space_token` VARCHAR(255) DEFAULT NULL,
`copy_pin_lifetime` INT(11) DEFAULT NULL,
`checksum_method` CHAR(1) DEFAULT NULL, -- Was checksum_method varchar(10), char(1) already in reality
`bring_online` INT(11) DEFAULT NULL,
`retry` INT(11) DEFAULT '0',
`retry_delay` INT(11) DEFAULT '0',
`job_metadata` TEXT, -- Was varchar(1024), already hit problems before because of size
PRIMARY KEY (`job_id`)
)
AS
SELECT job_id, job_state, reuse_job AS job_type, cancel_job,
source_se, dest_se, user_dn, cred_id, vo_name, reason,
submit_time, priority, submit_host,
max_time_in_queue, space_token, internal_job_params,
overwrite_flag, job_finished, source_space_token, copy_pin_lifetime,
checksum_method, bring_online, retry, retry_delay, job_metadata
FROM t_job
WHERE job_finished IS NULL;
RENAME TABLE t_job TO t_job_old;
RENAME TABLE t_job_new TO t_job;
ALTER TABLE t_job
ADD INDEX idx_vo_name (vo_name),
ADD INDEX idx_jobfinished (job_finished),
ADD INDEX idx_link (source_se, dest_se),
ADD INDEX idx_submission (submit_time, submit_host);
-- t_file
-- Dropped:
-- * logical_name VARCHAR(1100)
-- * symbolicName VARCHAR(255)
-- * error_scope VARCHAR(32)
-- * error_phase VARCHAR(32)
-- * reason_class VARCHAR(32)
-- * num_failures INT
-- * catalog_failures INT
-- * prestage_failures INT
-- * job_finished TIMESTAMP (redundant with t_job.job_finished)
CREATE TABLE t_file_new (
`file_id` BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT, -- Was file_id INT(11), as of today, 31 bits are required for a file_id
`file_index` INT(11) DEFAULT NULL,
`job_id` CHAR(36) NOT NULL,
`file_state` ENUM(
'STAGING', 'STARTED', 'SUBMITTED', 'READY', 'ACTIVE', 'FINISHED', 'FAILED', 'CANCELED', 'NOT_USED', 'ON_HOLD', 'ON_HOLD_STAGING'
) NOT NULL, -- Was file_state VARCHAR(32)
`transfer_host` VARCHAR(255) DEFAULT NULL, -- Was transferHost
`source_surl` VARCHAR(1100) DEFAULT NULL,
`dest_surl` VARCHAR(1100) DEFAULT NULL,
`source_se` VARCHAR(255) DEFAULT NULL,
`dest_se` VARCHAR(255) DEFAULT NULL,
`staging_host` VARCHAR(1024) DEFAULT NULL,
`reason` VARCHAR(2048) DEFAULT NULL,
`current_failures` INT(11) DEFAULT NULL,
`filesize` BIGINT DEFAULT NULL, -- Was filesize DOUBLE
`checksum` VARCHAR(100) DEFAULT NULL,
`finish_time` TIMESTAMP NULL DEFAULT NULL,
`start_time` TIMESTAMP NULL DEFAULT NULL,
`internal_file_params` VARCHAR(255) DEFAULT NULL,
`pid` INT(11) DEFAULT NULL,
`tx_duration` DOUBLE DEFAULT NULL,
`throughput` FLOAT DEFAULT NULL,
`retry` INT(11) DEFAULT '0',
`user_filesize` BIGINT DEFAULT NULL, -- Was DOUBLE
`file_metadata` TEXT, -- Was file_metadata VARCHAR(1024)
`selection_strategy` CHAR(32) DEFAULT NULL, -- Was VARCHAR(255)
`staging_start` TIMESTAMP NULL DEFAULT NULL,
`staging_finished` TIMESTAMP NULL DEFAULT NULL,
`bringonline_token` VARCHAR(255) DEFAULT NULL,
`retry_timestamp` TIMESTAMP NULL DEFAULT NULL,
`log_file` VARCHAR(2048) DEFAULT NULL,
`log_file_debug` TINYINT(1) DEFAULT NULL, -- Was INT
`hashed_id` INT(10) unsigned DEFAULT '0',
`vo_name` VARCHAR(50) DEFAULT NULL,
`activity` VARCHAR(255) DEFAULT 'default',
`transferred` BIGINT DEFAULT '0', -- Was DOUBLE
CONSTRAINT `job_id` FOREIGN KEY (`job_id`) REFERENCES `t_job` (`job_id`)
)
AS
SELECT file_id, file_index, job_id, file_state,
transferHost AS transfer_host, source_surl, dest_surl, source_se, dest_se,
agent_dn AS staging_host, reason, current_failures, filesize, checksum,
finish_time, start_time, internal_file_params,
pid, tx_duration, throughput, retry, user_filesize,
file_metadata, selection_strategy, staging_start, staging_finished,
bringonline_token, retry_timestamp,
t_log_file AS log_file, t_log_file_debug AS t_log_file_debug, hashed_id, vo_name, activity, transferred
FROM t_file
WHERE job_finished IS NULL;
RENAME TABLE t_file TO t_file_old;
RENAME TABLE t_file_new TO t_file;
ALTER TABLE t_file
ADD INDEX idx_job_id (job_id),
ADD INDEX idx_activity (vo_name, activity),
ADD INDEX idx_state_host (file_state, transfer_host),
ADD INDEX idx_link_state_vo (source_se, dest_se, file_state, vo_name),
ADD INDEX idx_finish_time (finish_time),
ADD INDEX idx_staging (file_state, vo_name, source_se);
--
-- Need to re-create also t_file_retry_errors pointing to the new table
--
CREATE TABLE t_file_retry_errors_new (
`file_id` BIGINT UNSIGNED NOT NULL,
`attempt` INTEGER NOT NULL,
`datetime` TIMESTAMP NULL DEFAULT NULL,
`reason` VARCHAR(2048),
CONSTRAINT PRIMARY KEY (`file_id`, `attempt`),
CONSTRAINT FOREIGN KEY (`file_id`) REFERENCES `t_file` (`file_id`) ON DELETE CASCADE
);
RENAME TABLE t_file_retry_errors TO t_file_retry_errors_old;
RENAME TABLE t_file_retry_errors_new TO t_file_retry_errors;
--
-- Same goes for t_file_share_config
--
CREATE TABLE t_file_share_config_new (
`file_id` BIGINT UNSIGNED NOT NULL,
`source` VARCHAR(150) NOT NULL,
`destination` VARCHAR(150) NOT NULL,
`vo` VARCHAR(100) NOT NULL,
CONSTRAINT PRIMARY KEY (`file_id`, `source`, `destination`, `vo`),
CONSTRAINT FOREIGN KEY (`source`, `destination`, `vo`) REFERENCES `t_share_config` (`source`, `destination`, `vo`) ON DELETE CASCADE,
CONSTRAINT FOREIGN KEY (`file_id`) REFERENCES `t_file` (`file_id`) ON DELETE CASCADE
);
RENAME TABLE t_file_share_config TO t_file_share_config_old;
RENAME TABLE t_file_share_config_new TO t_file_share_config;
--
-- t_dm needs to point to the new t_job
--
ALTER TABLE t_dm
DROP FOREIGN KEY t_dm_ibfk_1;
ALTER TABLE t_dm
ADD CONSTRAINT `fk_job_id` FOREIGN KEY (`job_id`) REFERENCES `t_job` (`job_id`);
--
-- Archive tables need to match the new schema
--
RENAME TABLE t_file_backup TO t_file_backup_old;
RENAME TABLE t_dm_backup TO t_dm_backup_old;
RENAME TABLE t_job_backup TO t_job_backup_old;
CREATE TABLE t_file_backup ENGINE = ARCHIVE AS (SELECT * FROM t_file WHERE NULL);
CREATE TABLE t_dm_backup ENGINE = ARCHIVE AS (SELECT * FROM t_dm WHERE NULL);
CREATE TABLE t_job_backup ENGINE = ARCHIVE AS (SELECT * FROM t_job WHERE NULL);
--
-- View for files that are to be staged, but haven't been requested
--
CREATE VIEW v_staging AS
SELECT q.job_id, q.file_id, q.hashed_id, q.vo_name, q.source_se, q.file_state, q.source_surl
FROM t_file q LEFT JOIN t_file s ON
q.source_surl = s.source_surl AND q.vo_name = s.vo_name AND s.source_se = q.source_se AND
s.file_state='STARTED'
WHERE q.file_state='STAGING' AND s.file_state IS NULL;
--
-- Change t_bas_ses primary key
--
ALTER TABLE t_bad_ses
DROP PRIMARY KEY,
ADD PRIMARY KEY(se, vo);
-- DROP TABLE t_file_share_config_old;
-- DROP TABLE t_file_retry_errors_old;
-- DROP TABLE t_file_old;
-- DROP TABLE t_job_old;
--
-- Unused
--
DROP TABLE t_server_sanity;
INSERT INTO t_schema_vers (major, minor, patch, message)
VALUES (3, 0, 0, 'FTS-599, FTS-815, FTS-824, FTS-629, FTS-859 diff');
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
assets/fts/mysql/fts-diff-3.0.0.sql 0 → 100644
+ 291
− 0
View file @ 9daa5bc2
SET default_storage_engine=InnoDB;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
ALTER TABLE t_activity_share_config
CHANGE COLUMN `activity_share` `activity_share` VARCHAR(1024) NOT NULL;
-- Per https://gitlab.cern.ch/fts/fts3/blob/develop/src/db/schema/unused.md
-- These tables are fairly small, so an in-place modification is reasonable
ALTER TABLE t_optimize
DROP COLUMN `file_id`,
DROP COLUMN `timeout`,
DROP COLUMN `buffer`,
DROP COLUMN `filesize`;
ALTER TABLE t_link_config
DROP COLUMN `placeholder1`,
DROP COLUMN `placeholder2`,
DROP COLUMN `placeholder3`,
DROP COLUMN `NO_TX_ACTIVITY_TO`;
-- For better performance on the web overview
ALTER TABLE t_optimizer_evolution
DROP INDEX t_optimizer_source_and_dest,
ADD INDEX idx_optimizer_evolution(`source_se`, `dest_se`, `datetime`);
-- Reduce size of dlg_id
CREATE TABLE t_credential_cache_new (
`dlg_id` CHAR(16) NOT NULL,
`dn` VARCHAR(255),
`cert_request` LONGTEXT,
`priv_key` LONGTEXT,
`voms_attrs` LONGTEXT,
PRIMARY KEY (dlg_id, dn)
)
AS
SELECT dlg_id, dn, cert_request, priv_key, voms_attrs
FROM t_credential_cache;
RENAME TABLE t_credential_cache TO t_credential_cache_old;
RENAME TABLE t_credential_cache_new TO t_credential_cache;
CREATE TABLE t_credential_new (
`dlg_id` CHAR(16) NOT NULL,
`dn` VARCHAR(255),
`proxy` LONGTEXT,
`voms_attrs` LONGTEXT,
`termination_time` TIMESTAMP NOT NULL,
PRIMARY KEY (dlg_id, dn),
INDEX (termination_time)
)
AS
SELECT dlg_id, dn, proxy, voms_attrs, termination_time
FROM t_credential;
RENAME TABLE t_credential TO t_credential_old;
RENAME TABLE t_credential_new TO t_credential;
-- DROP TABLE t_credential_cache_old;
-- DROP TABLE t_credential_old;
-- Unused + FTS-599 + FTS-617
-- These tables can be quite big, so an in-place modification would take
-- a long time.
-- Thus, we update the schema in two steps:
-- * Replicate a modified table with the data cloned
-- * Add indexes
-- t_job
-- t_file has a foreign key on this, so need to do it first.
-- Dropped fields
-- * job_params varchar(255), unused
-- * agent_dn varchar(1024), unused
-- * user_cred varchar(255), pointless
-- * voms_cred longtext, depends on cred_id
-- * storage_class varchar(255), unused
-- * myproxy_server varchar(255), unused
-- * source_token_description varchar(255), unused
-- * fail_nearline char(1), unused
-- * configuration_count (int), unused
-- * finish_time (timestamp), redundant with job_finished
CREATE TABLE t_job_new (
`job_id` CHAR(36) NOT NULL,
`job_state` ENUM(
'STAGING', 'SUBMITTED', 'READY', 'ACTIVE', 'FINISHED', 'FAILED', 'FINISHEDDIRTY', 'CANCELED', 'DELETE'
) NOT NULL, -- Was job_state varchar(32)
`job_type` CHAR(1) DEFAULT NULL, -- Was reuse_job varchar(3)
`cancel_job` CHAR(1) DEFAULT NULL,
`source_se` VARCHAR(255) DEFAULT NULL,
`dest_se` VARCHAR(255) DEFAULT NULL,
`user_dn` VARCHAR(1024) DEFAULT NULL,
`cred_id` CHAR(16) DEFAULT NULL, -- Was cred_id varchar(100), actually 16 hex digits are stored
`vo_name` VARCHAR(50) DEFAULT NULL,
`reason` VARCHAR(2048) DEFAULT NULL,
`submit_time` TIMESTAMP NULL DEFAULT NULL,
`priority` INT(11) DEFAULT '3',
`submit_host` VARCHAR(255) DEFAULT NULL,
`max_time_in_queue` INT(11) DEFAULT NULL,
`space_token` VARCHAR(255) DEFAULT NULL,
`internal_job_params` VARCHAR(255) DEFAULT NULL,
`overwrite_flag` CHAR(1) DEFAULT NULL,
`job_finished` TIMESTAMP NULL DEFAULT NULL,
`source_space_token` VARCHAR(255) DEFAULT NULL,
`copy_pin_lifetime` INT(11) DEFAULT NULL,
`checksum_method` CHAR(1) DEFAULT NULL, -- Was checksum_method varchar(10), char(1) already in reality
`bring_online` INT(11) DEFAULT NULL,
`retry` INT(11) DEFAULT '0',
`retry_delay` INT(11) DEFAULT '0',
`job_metadata` TEXT, -- Was varchar(1024), already hit problems before because of size
PRIMARY KEY (`job_id`)
)
AS
SELECT job_id, job_state, reuse_job AS job_type, cancel_job,
source_se, dest_se, user_dn, cred_id, vo_name, reason,
submit_time, priority, submit_host,
max_time_in_queue, space_token, internal_job_params,
overwrite_flag, job_finished, source_space_token, copy_pin_lifetime,
checksum_method, bring_online, retry, retry_delay, job_metadata
FROM t_job;
RENAME TABLE t_job TO t_job_old;
RENAME TABLE t_job_new TO t_job;
ALTER TABLE t_job
ADD INDEX idx_vo_name (vo_name),
ADD INDEX idx_jobfinished (job_finished),
ADD INDEX idx_link (source_se, dest_se),
ADD INDEX idx_submission (submit_time, submit_host);
-- t_file
-- Dropped:
-- * logical_name VARCHAR(1100)
-- * symbolicName VARCHAR(255)
-- * error_scope VARCHAR(32)
-- * error_phase VARCHAR(32)
-- * reason_class VARCHAR(32)
-- * num_failures INT
-- * catalog_failures INT
-- * prestage_failures INT
-- * job_finished TIMESTAMP (redundant with t_job.job_finished)
CREATE TABLE t_file_new (
`file_id` BIGINT UNSIGNED PRIMARY KEY AUTO_INCREMENT, -- Was file_id INT(11), as of today, 31 bits are required for a file_id
`file_index` INT(11) DEFAULT NULL,
`job_id` CHAR(36) NOT NULL,
`file_state` ENUM(
'STAGING', 'STARTED', 'SUBMITTED', 'READY', 'ACTIVE', 'FINISHED', 'FAILED', 'CANCELED', 'NOT_USED', 'ON_HOLD', 'ON_HOLD_STAGING'
) NOT NULL, -- Was file_state VARCHAR(32)
`transfer_host` VARCHAR(255) DEFAULT NULL, -- Was transferHost
`source_surl` VARCHAR(1100) DEFAULT NULL,
`dest_surl` VARCHAR(1100) DEFAULT NULL,
`source_se` VARCHAR(255) DEFAULT NULL,
`dest_se` VARCHAR(255) DEFAULT NULL,
`staging_host` VARCHAR(1024) DEFAULT NULL,
`reason` VARCHAR(2048) DEFAULT NULL,
`current_failures` INT(11) DEFAULT NULL,
`filesize` BIGINT DEFAULT NULL, -- Was filesize DOUBLE
`checksum` VARCHAR(100) DEFAULT NULL,
`finish_time` TIMESTAMP NULL DEFAULT NULL,
`start_time` TIMESTAMP NULL DEFAULT NULL,
`internal_file_params` VARCHAR(255) DEFAULT NULL,
`pid` INT(11) DEFAULT NULL,
`tx_duration` DOUBLE DEFAULT NULL,
`throughput` FLOAT DEFAULT NULL,
`retry` INT(11) DEFAULT '0',
`user_filesize` BIGINT DEFAULT NULL, -- Was DOUBLE
`file_metadata` TEXT, -- Was file_metadata VARCHAR(1024)
`selection_strategy` CHAR(32) DEFAULT NULL, -- Was VARCHAR(255)
`staging_start` TIMESTAMP NULL DEFAULT NULL,
`staging_finished` TIMESTAMP NULL DEFAULT NULL,
`bringonline_token` VARCHAR(255) DEFAULT NULL,
`retry_timestamp` TIMESTAMP NULL DEFAULT NULL,
`log_file` VARCHAR(2048) DEFAULT NULL,
`log_file_debug` TINYINT(1) DEFAULT NULL, -- Was INT
`hashed_id` INT(10) unsigned DEFAULT '0',
`vo_name` VARCHAR(50) DEFAULT NULL,
`activity` VARCHAR(255) DEFAULT 'default',
`transferred` BIGINT DEFAULT '0', -- Was DOUBLE
CONSTRAINT `job_id` FOREIGN KEY (`job_id`) REFERENCES `t_job` (`job_id`)
)
AS
SELECT file_id, file_index, job_id, file_state,
transferHost AS transfer_host, source_surl, dest_surl, source_se, dest_se,
agent_dn AS staging_host, reason, current_failures, filesize, checksum,
finish_time, start_time, internal_file_params,
pid, tx_duration, throughput, retry, user_filesize,
file_metadata, selection_strategy, staging_start, staging_finished,
bringonline_token, retry_timestamp,
t_log_file AS log_file, t_log_file_debug AS t_log_file_debug, hashed_id, vo_name, activity, transferred
FROM t_file;
RENAME TABLE t_file TO t_file_old;
RENAME TABLE t_file_new TO t_file;
ALTER TABLE t_file
ADD INDEX idx_job_id (job_id),
ADD INDEX idx_activity (vo_name, activity),
ADD INDEX idx_state_host (file_state, transfer_host),
ADD INDEX idx_link_state_vo (source_se, dest_se, file_state, vo_name),
ADD INDEX idx_finish_time (finish_time),
ADD INDEX idx_staging (file_state, vo_name, source_se);
--
-- Need to re-create also t_file_retry_errors pointing to the new table
--
CREATE TABLE t_file_retry_errors_new (
`file_id` BIGINT UNSIGNED NOT NULL,
`attempt` INTEGER NOT NULL,
`datetime` TIMESTAMP NULL DEFAULT NULL,
`reason` VARCHAR(2048),
CONSTRAINT PRIMARY KEY (`file_id`, `attempt`),
CONSTRAINT FOREIGN KEY (`file_id`) REFERENCES `t_file` (`file_id`) ON DELETE CASCADE
)
AS
SELECT file_id, attempt, datetime, reason
FROM t_file_retry_errors;
RENAME TABLE t_file_retry_errors TO t_file_retry_errors_old;
RENAME TABLE t_file_retry_errors_new TO t_file_retry_errors;
--
-- Same goes for t_file_share_config
--
CREATE TABLE t_file_share_config_new (
`file_id` BIGINT UNSIGNED NOT NULL,
`source` VARCHAR(150) NOT NULL,
`destination` VARCHAR(150) NOT NULL,
`vo` VARCHAR(100) NOT NULL,
CONSTRAINT PRIMARY KEY (`file_id`, `source`, `destination`, `vo`),
CONSTRAINT FOREIGN KEY (`source`, `destination`, `vo`) REFERENCES `t_share_config` (`source`, `destination`, `vo`) ON DELETE CASCADE,
CONSTRAINT FOREIGN KEY (`file_id`) REFERENCES `t_file` (`file_id`) ON DELETE CASCADE
)
AS
SELECT file_id, source, destination, vo
FROM t_file_share_config;
RENAME TABLE t_file_share_config TO t_file_share_config_old;
RENAME TABLE t_file_share_config_new TO t_file_share_config;
--
-- t_dm needs to point to the new t_job
--
ALTER TABLE t_dm
DROP FOREIGN KEY t_dm_ibfk_1;
ALTER TABLE t_dm
ADD CONSTRAINT `fk_job_id` FOREIGN KEY (`job_id`) REFERENCES `t_job` (`job_id`);
--
-- Archive tables need to match the new schema
--
RENAME TABLE t_file_backup TO t_file_backup_old;
RENAME TABLE t_dm_backup TO t_dm_backup_old;
RENAME TABLE t_job_backup TO t_job_backup_old;
CREATE TABLE t_file_backup ENGINE = ARCHIVE AS (SELECT * FROM t_file WHERE NULL);
CREATE TABLE t_dm_backup ENGINE = ARCHIVE AS (SELECT * FROM t_dm WHERE NULL);
CREATE TABLE t_job_backup ENGINE = ARCHIVE AS (SELECT * FROM t_job WHERE NULL);
--
-- View for files that are to be staged, but haven't been requested
--
CREATE VIEW v_staging AS
SELECT q.job_id, q.file_id, q.hashed_id, q.vo_name, q.source_se, q.file_state, q.source_surl
FROM t_file q LEFT JOIN t_file s ON
q.source_surl = s.source_surl AND q.vo_name = s.vo_name AND s.source_se = q.source_se AND
s.file_state='STARTED'
WHERE q.file_state='STAGING' AND s.file_state IS NULL;
--
-- Change t_bas_ses primary key
--
ALTER TABLE t_bad_ses
DROP PRIMARY KEY,
ADD PRIMARY KEY(se, vo);
-- DROP TABLE t_file_share_config_old;
-- DROP TABLE t_file_retry_errors_old;
-- DROP TABLE t_file_old;
-- DROP TABLE t_job_old;
--
-- Unused
--
DROP TABLE t_server_sanity;
INSERT INTO t_schema_vers (major, minor, patch, message)
VALUES (3, 0, 0, 'FTS-599, FTS-815, FTS-824, FTS-629, FTS-859 diff');
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
assets/fts/mysql/fts-diff-4.0.0.sql 0 → 100644
+ 125
− 0
View file @ 9daa5bc2
SET default_storage_engine=INNODB;
--
-- Storage configuration
--
RENAME TABLE t_se TO t_se_old;
CREATE TABLE t_se (
`storage` VARCHAR(150) NOT NULL PRIMARY KEY,
`site` VARCHAR(45),
`metadata` TEXT NULL,
`ipv6` BOOL NULL,
`udt` BOOL NULL,
`debug_level` INT NULL,
`inbound_max_active` INT,
`inbound_max_throughput` FLOAT,
`outbound_max_active` INT,
`outbound_max_throughput` FLOAT
)
AS
SELECT o.source_se AS storage, NULL AS site, NULL AS metadata,
o.ipv6 = 'on' AS ipv6, o.udt = 'on' AS udt, d.debug_level AS debug_level,
NULL AS inbound_max_active, NULL AS inbound_max_throughput,
NULL AS outbound_max_active, NULL AS outbound_max_throughput
FROM t_optimize o LEFT JOIN t_debug d ON o.source_se = d.source_se
WHERE o.source_se IS NOT NULL
UNION
SELECT o.dest_se, NULL, NULL, o.ipv6 = 'on', o.udt = 'on', d.debug_level, NULL, NULL, NULL, NULL
FROM t_optimize o LEFT JOIN t_debug d ON o.dest_se = d.dest_se
WHERE o.dest_se IS NOT NULL
UNION
SELECT d.source_se, NULL, NULL, NULL, NULL, d.debug_level, NULL, NULL, NULL, NULL
FROM t_debug d
WHERE d.source_se IS NOT NULL AND d.source_se != ""
UNION
SELECT d.dest_se, NULL, NULL, NULL, NULL, d.debug_level, NULL, NULL, NULL, NULL
FROM t_debug d
WHERE d.dest_se IS NOT NULL AND d.dest_se != ""
UNION
(SELECT '*', NULL, NULL, NULL, NULL, NULL, max_per_se, NULL, max_per_se, NULL
FROM t_server_config
WHERE vo_name IN (NULL, '*', '')
LIMIT 1);
--
-- Link configuration
--
RENAME TABLE t_link_config TO t_link_config_old;
CREATE TABLE t_link_config (
source_se VARCHAR(150) NOT NULL,
dest_se VARCHAR(150) NOT NULL,
symbolic_name VARCHAR(150) NOT NULL UNIQUE,
min_active INT NULL,
max_active INT NULL,
optimizer_mode INT NULL,
tcp_buffer_size INT,
nostreams INT,
PRIMARY KEY(source_se, dest_se)
)
AS
SELECT o.source_se AS source_se, o.dest_se AS dest_se, CONCAT(source_se, "-", dest_se) AS symbolic_name,
o.min_active AS min_active, o.max_active AS max_active, om.mode_opt AS optimizer_mode,
l.tcp_buffer_size AS tcp_buffer_size, l.nostreams AS nostreams
FROM t_optimize_mode om, t_optimize_active o
LEFT JOIN t_link_config_old l ON l.source = o.source_se AND l.destination = o.dest_se
WHERE o.fixed='on'
UNION
(SELECT l.source, l.destination, l.symbolicName, NULL, NULL, NULL, l.tcp_buffer_size, l.nostreams
FROM t_link_config_old l
WHERE NOT EXISTS (SELECT 1 FROM t_optimize_active oa WHERE oa.source_se = l.source AND oa.dest_se = l.destination)
)
UNION
(SELECT '*', '*', '*', 2, s.max_per_link, om.mode_opt, NULL, s.global_tcp_stream
FROM t_optimize_mode om, t_server_config s
WHERE vo_name IN ('*', '') OR vo_name IS NULL
LIMIT 1);
--
-- Optimizer status
--
CREATE TABLE t_optimizer (
`source_se` VARCHAR(150) NOT NULL,
`dest_se` VARCHAR(150) NOT NULL,
`datetime` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`ema` DOUBLE DEFAULT 0,
`active` INT DEFAULT 2,
`nostreams` INT DEFAULT 1,
PRIMARY KEY(`source_se`, `dest_se`)
)
AS
SELECT
o.source_se, o.dest_se, o.datetime, o.ema, o.active, s.nostreams
FROM t_optimize_active o
LEFT JOIN t_optimize_streams s ON s.source_se = o.source_se and s.dest_se = o.dest_se;
--
-- Drop obsolete fields
--
ALTER TABLE t_server_config
DROP COLUMN max_per_link,
DROP COLUMN max_per_se,
DROP COLUMN global_tcp_stream;
--
-- Drop link between t_file and t_share_config
--
DROP TABLE t_file_share_config;
ALTER TABLE t_share_config
DROP FOREIGN KEY t_share_config_fk;
ALTER TABLE t_share_config
ADD CONSTRAINT t_share_config_fk FOREIGN KEY (source, destination) REFERENCES t_link_config (source_se, dest_se);
--
-- Cleanup
--
-- DROP TABLE t_group_members;
-- DROP TABLE t_se_old;
-- DROP TABLE t_link_config_old;
-- DROP TABLE t_debug;
-- DROP TABLE t_optimize_mode;
-- DROP TABLE t_optimize;
-- DROP TABLE t_optimize_streams;
-- DROP TABLE t_optimize_active;
INSERT INTO t_schema_vers (major, minor, patch, message)
VALUES (4, 0, 0, 'FTS-894 diff');
assets/fts/mysql/fts-diff-4.0.1.sql 0 → 100644
+ 10
− 0
View file @ 9daa5bc2
--
-- FTS3 Schema 4.0.1
-- [FTS-1042] t_credential.termination_time should not have the "ON UPDATE" clause
--
ALTER TABLE t_credential
MODIFY COLUMN termination_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP;
INSERT INTO t_schema_vers (major, minor, patch, message)
VALUES (4, 0, 1, 'FTS-1042 diff');
assets/fts/mysql/fts-diff-5.0.0.sql 0 → 100644
+ 91
− 0
View file @ 9daa5bc2
--
-- FTS3 Schema 5.0.0
-- [FTS-1318] Study FTS schema and propose optimisations
-- [FTS-1239] Revise the t_dm schema
-- [FTS-1329] Make t_file and t_file_backup reason UTF-8
-- [FTS-1201] De-normalize priority in DB
-- [FTS-1353] Avoid submitting multiple transfers to the same destination
--
ALTER TABLE `t_job`
ADD INDEX `idx_jobtype` (`job_type`);
ALTER TABLE `t_file`
ADD INDEX `idx_state` (`file_state`);
ALTER TABLE `t_file`
ADD INDEX `idx_host` (`transfer_host`);
ALTER TABLE `t_optimizer_evolution`
ADD INDEX `idx_datetime` ( `datetime`);
ALTER TABLE `t_file`
MODIFY reason varchar(2048) CHARACTER SET utf8;
ALTER TABLE `t_file_backup`
MODIFY reason varchar(2048) CHARACTER SET utf8;
ALTER TABLE `t_file_retry_errors`
MODIFY reason varchar(2048) CHARACTER SET utf8;
ALTER TABLE `t_file_retry_errors`
ADD INDEX `idx_datetime` ( `datetime`);
ALTER TABLE `t_file`
ADD COLUMN `priority` int(11) DEFAULT '3';
UPDATE `t_job`
SET `priority`=3 where `priority`!=3;
ALTER TABLE `t_file`
ADD COLUMN `dest_surl_uuid` char(36) DEFAULT NULL;
ALTER TABLE `t_file`
ADD UNIQUE KEY `dest_surl_uuid` (`dest_surl_uuid`);
CREATE TABLE `t_dm_new` (
`file_id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`job_id` char(36) NOT NULL,
`file_state` varchar(32) NOT NULL,
`dmHost` varchar(150) DEFAULT NULL,
`source_surl` varchar(900) DEFAULT NULL,
`dest_surl` varchar(900) DEFAULT NULL,
`source_se` varchar(150) DEFAULT NULL,
`dest_se` varchar(150) DEFAULT NULL,
`error_scope` varchar(32) DEFAULT NULL,
`error_phase` varchar(32) DEFAULT NULL,
`reason` varchar(2048) CHARACTER SET utf8 DEFAULT NULL,
`checksum` varchar(100) DEFAULT NULL,
`finish_time` timestamp NULL DEFAULT NULL,
`start_time` timestamp NULL DEFAULT NULL,
`internal_file_params` varchar(255) DEFAULT NULL,
`job_finished` timestamp NULL DEFAULT NULL,
`pid` int(11) DEFAULT NULL,
`tx_duration` double DEFAULT NULL,
`retry` int(11) DEFAULT '0',
`user_filesize` double DEFAULT NULL,
`file_metadata` varchar(255) DEFAULT NULL,
`activity` varchar(255) DEFAULT 'default',
`selection_strategy` varchar(255) DEFAULT NULL,
`dm_start` timestamp NULL DEFAULT NULL,
`dm_finished` timestamp NULL DEFAULT NULL,
`dm_token` varchar(255) DEFAULT NULL,
`retry_timestamp` timestamp NULL DEFAULT NULL,
`wait_timestamp` timestamp NULL DEFAULT NULL,
`wait_timeout` int(11) DEFAULT NULL,
`hashed_id` int(10) unsigned DEFAULT '0',
`vo_name` varchar(100) DEFAULT NULL,
PRIMARY KEY (`file_id`),
CONSTRAINT `fk_dmjob_id` FOREIGN KEY (`job_id`) REFERENCES `t_job` (`job_id`)
)
AS
SELECT file_id, job_id, file_state, dmHost, source_surl, dest_surl, source_se, dest_se, NULL as error_scope, NULL as error_phase, reason, checksum, finish_time, start_time, NULL as internal_file_params,
job_finished, NULL as pid, tx_duration, retry, user_filesize, file_metadata, activity, NULL as selection_strategy, NULL as dm_start, NULL as dm_finished, dm_token, retry_timestamp, wait_timestamp, wait_timeout, hashed_id, vo_name
FROM t_dm;
RENAME TABLE t_dm TO t_dm_old;
RENAME TABLE t_dm_new TO t_dm;
ALTER TABLE t_dm
ADD INDEX dm_job_id (job_id);
--
-- Archive tables need to match the new schema
--
RENAME TABLE t_dm_backup TO t_dm_backup_old;
CREATE TABLE t_dm_backup ENGINE = ARCHIVE AS (SELECT * FROM t_dm WHERE NULL);
DROP TABLE t_dm_old;
DROP TABLE t_dm_backup_old;
INSERT INTO t_schema_vers (major, minor, patch, message)
VALUES (5, 0, 0, 'FTS-1318 diff');
assets/fts/mysql/fts-diff-6.0.0.sql 0 → 100644
+ 49
− 0
View file @ 9daa5bc2
--
-- FTS3 Schema 6.0.0
-- Schema changes for QoS daemon and OIDC integration
--
ALTER TABLE `t_file`
ADD COLUMN `archive_start_time` timestamp NULL DEFAULT NULL,
ADD COLUMN `archive_finish_time` timestamp NULL DEFAULT NULL;
ALTER TABLE `t_file`
MODIFY COLUMN file_state enum('STAGING','ARCHIVING','QOS_TRANSITION','QOS_REQUEST_SUBMITTED','STARTED','SUBMITTED','READY','ACTIVE','FINISHED','FAILED','CANCELED','NOT_USED','ON_HOLD','ON_HOLD_STAGING') NOT NULL;
ALTER TABLE `t_file_backup`
ADD COLUMN `archive_start_time` timestamp NULL DEFAULT NULL,
ADD COLUMN `archive_finish_time` timestamp NULL DEFAULT NULL;
ALTER TABLE `t_file_backup`
MODIFY COLUMN file_state enum('STAGING','ARCHIVING','QOS_TRANSITION','QOS_REQUEST_SUBMITTED','STARTED','SUBMITTED','READY','ACTIVE','FINISHED','FAILED','CANCELED','NOT_USED','ON_HOLD','ON_HOLD_STAGING') NOT NULL;
ALTER TABLE `t_job`
ADD COLUMN `target_qos` varchar(255) DEFAULT NULL,
ADD COLUMN `archive_timeout` int(11) DEFAULT NULL;
ALTER TABLE `t_job`
MODIFY COLUMN job_state enum('STAGING','ARCHIVING','QOS_TRANSITION','QOS_REQUEST_SUBMITTED','SUBMITTED','READY','ACTIVE','FINISHED','FAILED','FINISHEDDIRTY','CANCELED','DELETE') NOT NULL;
ALTER TABLE `t_job_backup`
ADD COLUMN `target_qos` varchar(255) DEFAULT NULL,
ADD COLUMN `archive_timeout` int(11) DEFAULT NULL;
ALTER TABLE `t_job_backup`
MODIFY COLUMN job_state enum('STAGING','ARCHIVING','QOS_TRANSITION','QOS_REQUEST_SUBMITTED','SUBMITTED','READY','ACTIVE','FINISHED','FAILED','FINISHEDDIRTY','CANCELED','DELETE') NOT NULL;
CREATE TABLE t_oauth2_providers (
`provider_url` VARCHAR(250) NOT NULL,
`provider_jwk` VARCHAR(1000) NOT NULL,
PRIMARY KEY(`provider_url`)
);
ALTER TABLE `t_cloudStorage`
MODIFY `cloudStorage_name` varchar(150) NOT NULL;
ALTER TABLE `t_cloudStorageUser`
MODIFY `cloudStorage_name` varchar(150) NOT NULL;
ALTER TABLE `t_link_config`
ADD COLUMN `no_delegation` varchar(3) DEFAULT NULL;
ALTER TABLE `t_server_config`
ADD COLUMN `no_streaming` varchar(3) DEFAULT NULL;
INSERT INTO t_schema_vers (major, minor, patch, message)
VALUES (6, 0, 0, 'QoS daemon, OIDC integration and archiving monitoring diff');
assets/fts/mysql/fts-downgrade-5.0.0.sql 0 → 100644
+ 44
− 0
View file @ 9daa5bc2
--
-- Script to downgrade from FTS3 Schema 5.0.0 to the previous schema
--
ALTER TABLE `t_job`
DROP INDEX `idx_jobtype`;
ALTER TABLE `t_file`
DROP INDEX `idx_state`;
ALTER TABLE `t_file`
DROP INDEX `idx_host`;
ALTER TABLE `t_optimizer_evolution`
DROP INDEX `idx_datetime`;
ALTER TABLE `t_file`
MODIFY reason varchar(2048) CHARACTER SET latin1;
ALTER TABLE `t_file_backup`
MODIFY reason varchar(2048) CHARACTER SET latin1;
ALTER TABLE `t_dm`
MODIFY reason varchar(2048) CHARACTER SET latin1;
ALTER TABLE `t_dm_backup`
MODIFY reason varchar(2048) CHARACTER SET latin1;
ALTER TABLE `t_file_retry_errors`
MODIFY reason varchar(2048) CHARACTER SET latin1;
ALTER TABLE `t_file_retry_errors`
DROP INDEX `idx_datetime`;
ALTER TABLE `t_file`
DROP `priority`;
ALTER TABLE `t_file`
DROP UNIQUE KEY `dest_surl_uuid`;
ALTER TABLE `t_file`
DROP `dest_surl_uuid`;
DROP TABLE t_dm;
RENAME TABLE t_dm_old TO t_dm;
ALTER TABLE t_dm
DROP INDEX dm_job_id;
--
-- Archive tables need to match the new schema
--
DROP TABLE t_dm_backup;
RENAME TABLE t_dm_backup_old TO t_dm_backup;
UPDATE t_schema_vers SET MAJOR=4, PATCH=1, MESSAGE='DOWNGRADE from 5.0.0' where MAJOR = 5;
assets/fts/mysql/fts-downgrade-6.0.0.sql 0 → 100644
+ 50
− 0
View file @ 9daa5bc2
--
-- Script to downgrade from FTS3 Schema 6.0.0 to the previous schema, 5.0.0
--
ALTER TABLE `t_job`
DROP COLUMN `target_qos`,
DROP COLUMN `archive_timeout`;
ALTER TABLE `t_job`
MODIFY COLUMN job_state enum('STAGING','SUBMITTED','READY','ACTIVE','FINISHED','FAILED','FINISHEDDIRTY','CANCELED','DELETE') NOT NULL;
ALTER TABLE `t_job_backup`
MODIFY COLUMN job_state enum('STAGING','SUBMITTED','READY','ACTIVE','FINISHED','FAILED','FINISHEDDIRTY','CANCELED','DELETE') NOT NULL;
ALTER TABLE `t_job_backup`
DROP COLUMN `target_qos`,
DROP COLUMN `archive_timeout`;
ALTER TABLE `t_file`
DROP COLUMN `archive_start_time`,
DROP COLUMN `archive_finish_time`;
ALTER TABLE `t_file`
MODIFY COLUMN file_state enum('STAGING','STARTED','SUBMITTED','READY','ACTIVE','FINISHED','FAILED','CANCELED','NOT_USED','ON_HOLD','ON_HOLD_STAGING') NOT NULL;
ALTER TABLE `t_file_backup`
DROP COLUMN `archive_start_time`,
DROP COLUMN `archive_finish_time`;
ALTER TABLE `t_file_backup`
MODIFY COLUMN file_state enum('STAGING','STARTED','SUBMITTED','READY','ACTIVE','FINISHED','FAILED','CANCELED','NOT_USED','ON_HOLD','ON_HOLD_STAGING') NOT NULL;
DROP TABLE t_oauth2_providers;
---
--- Manage foreign key on `cloudStorage_name` before changing var length
---
ALTER TABLE `t_cloudStorageUser`
DROP FOREIGN KEY `t_cloudStorageUser_ibfk_1`,
MODIFY `cloudStorage_name` varchar(36) NOT NULL;
ALTER TABLE `t_cloudStorage`
MODIFY `cloudStorage_name` varchar(50) NOT NULL;
ALTER TABLE `t_cloudStorageUser`
ADD CONSTRAINT `t_cloudStorageUser_ibfk_1` FOREIGN KEY (`cloudStorage_name`) REFERENCES `t_cloudStorage` (`cloudStorage_name`);
ALTER TABLE `t_link_config`
DROP COLUMN `no_delegation`;
ALTER TABLE `t_server_config`
DROP COLUMN `no_streaming`;
UPDATE t_schema_vers SET MAJOR=5, PATCH=0, MESSAGE='DOWNGRADE from 6.0.0' where MAJOR = 6;
assets/fts/mysql/fts-schema-1.0.0.sql 0 → 100644
+ 922
− 0
View file @ 9daa5bc2
This diff is collapsed.
assets/fts/mysql/fts-schema-2.0.0.sql 0 → 100644
+ 863
− 0
View file @ 9daa5bc2
This diff is collapsed.
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment