package cache
import (
"fmt"
"net/http"
"net/url"
"sync"
"time"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type Adapter interface {
GetDownloadURL() *url.URL
GetUploadURL() *url.URL
GetUploadHeaders() http.Header
GetGoCloudURL() *url.URL
GetUploadEnv() map[string]string
}
type Factory func(config *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error)
type FactoriesMap struct {
internal map[string]Factory
lock sync.RWMutex
}
func (m *FactoriesMap) Register(typeName string, factory Factory) error {
m.lock.Lock()
defer m.lock.Unlock()
if len(m.internal) == 0 {
m.internal = make(map[string]Factory)
}
_, ok := m.internal[typeName]
if ok {
return fmt.Errorf("adapter %q already registered", typeName)
}
m.internal[typeName] = factory
return nil
}
func (m *FactoriesMap) Find(typeName string) (Factory, error) {
m.lock.RLock()
defer m.lock.RUnlock()
factory := m.internal[typeName]
if factory == nil {
return nil, fmt.Errorf("factory for cache adapter %q was not registered", typeName)
}
return factory, nil
}
var factories = &FactoriesMap{}
func Factories() *FactoriesMap {
return factories
}
func CreateAdapter(cacheConfig *common.CacheConfig, timeout time.Duration, objectName string) (Adapter, error) {
create, err := Factories().Find(cacheConfig.Type)
if err != nil {
return nil, fmt.Errorf("cache factory not found: %w", err)
}
adapter, err := create(cacheConfig, timeout, objectName)
if err != nil {
return nil, fmt.Errorf("cache adapter could not be initialized: %w", err)
}
return adapter, nil
}
package azure
import (
"fmt"
"net/http"
"net/url"
"strings"
"time"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/cache"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type signedURLGenerator func(name string, options *signedURLOptions) (*url.URL, error)
type blobTokenGenerator func(name string, options *signedURLOptions) (string, error)
type azureAdapter struct {
timeout time.Duration
config *common.CacheAzureConfig
objectName string
generateSignedURL signedURLGenerator
blobTokenGenerator blobTokenGenerator
credentialsResolver credentialsResolver
}
func (a *azureAdapter) GetDownloadURL() *url.URL {
return a.presignURL(http.MethodGet)
}
func (a *azureAdapter) GetUploadURL() *url.URL {
return a.presignURL(http.MethodPut)
}
func (a *azureAdapter) GetUploadHeaders() http.Header {
httpHeaders := http.Header{}
httpHeaders.Set("Content-Type", "application/octet-stream")
httpHeaders.Set("x-ms-blob-type", "BlockBlob")
return httpHeaders
}
func (a *azureAdapter) GetGoCloudURL() *url.URL {
if a.config.ContainerName == "" {
logrus.Error("ContainerName can't be empty")
return nil
}
// Go Cloud omits the object name from the URL. Since object storage
// providers use the URL host for the bucket name, we attach the
// object name to avoid having to pass another parameter.
raw := fmt.Sprintf("azblob://%s/%s", a.config.ContainerName, a.objectName)
u, err := url.Parse(raw)
if err != nil {
logrus.WithError(err).WithField("url", raw).Errorf("error parsing blob URL")
return nil
}
return u
}
func (a *azureAdapter) GetUploadEnv() map[string]string {
token := a.generateWriteToken()
if token == "" {
return map[string]string{}
}
return map[string]string{
"AZURE_STORAGE_ACCOUNT": a.config.AccountName,
"AZURE_STORAGE_SAS_TOKEN": token,
"AZURE_STORAGE_DOMAIN": a.config.StorageDomain,
}
}
func (a *azureAdapter) presignURL(method string) *url.URL {
credentials := a.getCredentials()
if credentials == nil {
return nil
}
u, err := a.generateSignedURL(a.objectName, &signedURLOptions{
ContainerName: a.config.ContainerName,
StorageDomain: a.config.StorageDomain,
Credentials: credentials,
Method: method,
Timeout: a.timeout,
})
if err != nil {
logrus.WithError(err).Errorf("error generating Azure pre-signed URL")
return nil
}
return u
}
func (a *azureAdapter) generateWriteToken() string {
credentials := a.getCredentials()
if credentials == nil {
return ""
}
t, err := a.blobTokenGenerator(a.objectName, &signedURLOptions{
ContainerName: a.config.ContainerName,
StorageDomain: a.config.StorageDomain,
Credentials: credentials,
Method: http.MethodPut,
Timeout: a.timeout,
})
if err != nil {
logrus.WithError(err).Errorf("error generating Azure SAS token")
return ""
}
return t
}
func (a *azureAdapter) getCredentials() *common.CacheAzureCredentials {
if a.config.ContainerName == "" {
logrus.Errorf("ContainerName can't be empty")
return nil
}
err := a.credentialsResolver.Resolve()
if err != nil {
logrus.WithError(err).Errorf("error resolving Azure credentials")
return nil
}
return a.credentialsResolver.Credentials()
}
func New(config *common.CacheConfig, timeout time.Duration, objectName string) (cache.Adapter, error) {
azure := config.Azure
if azure == nil {
return nil, fmt.Errorf("missing Azure configuration")
}
cr, err := credentialsResolverInitializer(azure)
if err != nil {
return nil, fmt.Errorf("error while initializing Azure credentials resolver: %w", err)
}
a := &azureAdapter{
config: azure,
timeout: timeout,
objectName: strings.TrimLeft(objectName, "/"),
credentialsResolver: cr,
generateSignedURL: presignedURL,
blobTokenGenerator: getSASToken,
}
return a, nil
}
func init() {
err := cache.Factories().Register("azure", New)
if err != nil {
panic(err)
}
}
package azure
import (
"errors"
"fmt"
"net/http"
"net/url"
"time"
"github.com/Azure/azure-storage-blob-go/azblob"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
const DefaultAzureServer = "blob.core.windows.net"
type signedURLOptions struct {
ContainerName string
StorageDomain string
Credentials *common.CacheAzureCredentials
Method string
Timeout time.Duration
}
func presignedURL(name string, o *signedURLOptions) (*url.URL, error) {
sas, err := getSASQueryParameters(name, o)
if err != nil {
return nil, err
}
domain := DefaultAzureServer
if o.StorageDomain != "" {
domain = o.StorageDomain
}
parts := azblob.BlobURLParts{
Scheme: "https",
Host: fmt.Sprintf("%s.%s", o.Credentials.AccountName, domain),
ContainerName: o.ContainerName,
BlobName: name,
SAS: sas,
}
u := parts.URL()
return &u, nil
}
func getSASToken(name string, o *signedURLOptions) (string, error) {
sas, err := getSASQueryParameters(name, o)
if err != nil {
return "", err
}
return sas.Encode(), nil
}
func getSASQueryParameters(name string, o *signedURLOptions) (azblob.SASQueryParameters, error) {
empty := azblob.SASQueryParameters{}
if o.Credentials.AccountName == "" {
return empty, errors.New("missing Azure storage account name")
}
if o.Credentials.AccountKey == "" {
return empty, errors.New("missing Azure storage account key")
}
credential, err := azblob.NewSharedKeyCredential(o.Credentials.AccountName, o.Credentials.AccountKey)
if err != nil {
return empty, fmt.Errorf("creating Azure signature: %w", err)
}
permissions := azblob.AccountSASPermissions{Read: true}
if o.Method == http.MethodPut {
permissions = azblob.AccountSASPermissions{Write: true}
}
// Set the desired SAS signature values and sign them with the
// shared key credentials to get the SAS query parameters.
// See https://docs.microsoft.com/en-us/rest/api/storageservices/create-service-sas
serviceSASValues := azblob.BlobSASSignatureValues{
Protocol: azblob.SASProtocolHTTPS, // Users MUST use HTTPS (not HTTP)
StartTime: time.Now().Add(-1 * time.Hour).UTC(),
ExpiryTime: time.Now().Add(o.Timeout).UTC(),
Permissions: permissions.String(),
ContainerName: o.ContainerName,
BlobName: name,
}
sas, err := serviceSASValues.NewSASQueryParameters(credential)
if err != nil {
return empty, fmt.Errorf("creating Azure SAS: %w", err)
}
return sas, nil
}
package azure
import (
"fmt"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type credentialsResolver interface {
Credentials() *common.CacheAzureCredentials
Resolve() error
}
type defaultCredentialsResolver struct {
config *common.CacheAzureConfig
credentials *common.CacheAzureCredentials
}
func (cr *defaultCredentialsResolver) Credentials() *common.CacheAzureCredentials {
return cr.credentials
}
func (cr *defaultCredentialsResolver) Resolve() error {
return cr.readCredentialsFromConfig()
}
func (cr *defaultCredentialsResolver) readCredentialsFromConfig() error {
if cr.config.AccountName == "" || cr.config.AccountKey == "" {
return fmt.Errorf("config for Azure present, but credentials are not configured")
}
cr.credentials.AccountName = cr.config.AccountName
cr.credentials.AccountKey = cr.config.AccountKey
return nil
}
func newDefaultCredentialsResolver(config *common.CacheAzureConfig) (*defaultCredentialsResolver, error) {
if config == nil {
return nil, fmt.Errorf("config can't be nil")
}
credentials := &defaultCredentialsResolver{
config: config,
credentials: &common.CacheAzureCredentials{},
}
return credentials, nil
}
var credentialsResolverInitializer = newDefaultCredentialsResolver
package cache
import (
"fmt"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
var createAdapter = CreateAdapter
func getCacheConfig(build *common.Build) *common.CacheConfig {
if build == nil || build.Runner == nil || build.Runner.Cache == nil {
return nil
}
return build.Runner.Cache
}
func generateBaseObjectName(build *common.Build, config *common.CacheConfig) string {
runnerSegment := ""
if !config.GetShared() {
runnerSegment = path.Join("runner", build.Runner.ShortDescription())
}
return path.Join(config.GetPath(), runnerSegment, "project", strconv.Itoa(build.JobInfo.ProjectID))
}
func generateObjectName(build *common.Build, config *common.CacheConfig, key string) (string, error) {
if key == "" {
return "", nil
}
basePath := generateBaseObjectName(build, config)
fullPath := path.Join(basePath, key)
// The typical concerns regarding the use of strings.HasPrefix to detect
// path traversal do not apply here. The detection here is made easier
// as we're dealing with URL paths, not filepaths and we're ensuring that
// the basepath has a final separator (the key can not be empty).
// TestGenerateObjectName contains path traversal tests.
if !strings.HasPrefix(fullPath, basePath+"/") {
return "", fmt.Errorf("computed cache path outside of project bucket. Please remove `../` from cache key")
}
return fullPath, nil
}
func buildAdapter(build *common.Build, key string) (Adapter, error) {
config := getCacheConfig(build)
if config == nil {
logrus.Warning("Cache config not defined. Skipping cache operation.")
return nil, nil
}
objectName, err := generateObjectName(build, config, key)
if err != nil {
logrus.WithError(err).Error("Error while generating cache bucket.")
return nil, nil
}
if objectName == "" {
logrus.Warning("Empty cache key. Skipping adapter selection.")
return nil, nil
}
return createAdapter(config, build.GetBuildTimeout(), objectName)
}
func onAdapter(build *common.Build, key string, handler func(adapter Adapter) interface{}) interface{} {
adapter, err := buildAdapter(build, key)
if err != nil {
logrus.WithError(err).Error("Could not create cache adapter")
}
if adapter == nil {
return nil
}
return handler(adapter)
}
func GetCacheDownloadURL(build *common.Build, key string) *url.URL {
return castToURL(func() interface{} {
return onAdapter(build, key, func(adapter Adapter) interface{} {
return adapter.GetDownloadURL()
})
})
}
func castToURL(handler func() interface{}) *url.URL {
result := handler()
u, ok := result.(*url.URL)
if !ok {
return nil
}
return u
}
func GetCacheUploadURL(build *common.Build, key string) *url.URL {
return castToURL(func() interface{} {
return onAdapter(build, key, func(adapter Adapter) interface{} {
return adapter.GetUploadURL()
})
})
}
func GetCacheUploadHeaders(build *common.Build, key string) http.Header {
result := onAdapter(build, key, func(adapter Adapter) interface{} {
return adapter.GetUploadHeaders()
})
h, ok := result.(http.Header)
if !ok {
return nil
}
return h
}
func GetCacheGoCloudURL(build *common.Build, key string) *url.URL {
return castToURL(func() interface{} {
return onAdapter(build, key, func(adapter Adapter) interface{} {
return adapter.GetGoCloudURL()
})
})
}
func GetCacheUploadEnv(build *common.Build, key string) map[string]string {
result := onAdapter(build, key, func(adapter Adapter) interface{} {
return adapter.GetUploadEnv()
})
m, ok := result.(map[string]string)
if !ok {
return nil
}
return m
}
package cache
import (
"fmt"
"sync"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type CredentialsAdapter interface {
GetCredentials() map[string]string
}
var credentialsFactories = &CredentialsFactoriesMap{}
func CredentialsFactories() *CredentialsFactoriesMap {
return credentialsFactories
}
type CredentialsFactory func(config *common.CacheConfig) (CredentialsAdapter, error)
type CredentialsFactoriesMap struct {
internal map[string]CredentialsFactory
lock sync.RWMutex
}
func (m *CredentialsFactoriesMap) Register(typeName string, factory CredentialsFactory) error {
m.lock.Lock()
defer m.lock.Unlock()
if len(m.internal) == 0 {
m.internal = make(map[string]CredentialsFactory)
}
_, ok := m.internal[typeName]
if ok {
return fmt.Errorf("credentials adapter %q already registered", typeName)
}
m.internal[typeName] = factory
return nil
}
func (m *CredentialsFactoriesMap) Find(typeName string) (CredentialsFactory, error) {
m.lock.RLock()
defer m.lock.RUnlock()
factory := m.internal[typeName]
if factory == nil {
return nil, fmt.Errorf("factory for credentials adapter %q not registered", typeName)
}
return factory, nil
}
func CreateCredentialsAdapter(cacheConfig *common.CacheConfig) (CredentialsAdapter, error) {
create, err := CredentialsFactories().Find(cacheConfig.Type)
if err != nil {
return nil, fmt.Errorf("credentials adapter factory not found: %w", err)
}
adapter, err := create(cacheConfig)
if err != nil {
return nil, fmt.Errorf("credentials adapter could not be initialized: %w", err)
}
return adapter, nil
}
package gcs
import (
"fmt"
"net/http"
"net/url"
"time"
"cloud.google.com/go/storage"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/cache"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type signedURLGenerator func(bucket string, name string, opts *storage.SignedURLOptions) (string, error)
type gcsAdapter struct {
timeout time.Duration
config *common.CacheGCSConfig
objectName string
generateSignedURL signedURLGenerator
credentialsResolver credentialsResolver
}
func (a *gcsAdapter) GetDownloadURL() *url.URL {
return a.presignURL(http.MethodGet, "")
}
func (a *gcsAdapter) GetUploadURL() *url.URL {
return a.presignURL(http.MethodPut, "application/octet-stream")
}
func (a *gcsAdapter) GetUploadHeaders() http.Header {
return nil
}
func (a *gcsAdapter) GetGoCloudURL() *url.URL {
return nil
}
func (a *gcsAdapter) GetUploadEnv() map[string]string {
return nil
}
func (a *gcsAdapter) presignURL(method string, contentType string) *url.URL {
err := a.credentialsResolver.Resolve()
if err != nil {
logrus.Errorf("error while resolving GCS credentials: %v", err)
return nil
}
credentials := a.credentialsResolver.Credentials()
var privateKey []byte
if credentials.PrivateKey != "" {
privateKey = []byte(credentials.PrivateKey)
}
if a.config.BucketName == "" {
logrus.Error("BucketName can't be empty")
return nil
}
rawURL, err := a.generateSignedURL(a.config.BucketName, a.objectName, &storage.SignedURLOptions{
GoogleAccessID: credentials.AccessID,
PrivateKey: privateKey,
Method: method,
Expires: time.Now().Add(a.timeout),
ContentType: contentType,
})
if err != nil {
logrus.Errorf("error while generating GCS pre-signed URL: %v", err)
return nil
}
URL, err := url.Parse(rawURL)
if err != nil {
logrus.Errorf("error while parsing generated URL: %v", err)
return nil
}
return URL
}
func New(config *common.CacheConfig, timeout time.Duration, objectName string) (cache.Adapter, error) {
gcs := config.GCS
if gcs == nil {
return nil, fmt.Errorf("missing GCS configuration")
}
cr, err := credentialsResolverInitializer(gcs)
if err != nil {
return nil, fmt.Errorf("error while initializing GCS credentials resolver: %w", err)
}
a := &gcsAdapter{
config: gcs,
timeout: timeout,
objectName: objectName,
generateSignedURL: storage.SignedURL,
credentialsResolver: cr,
}
return a, nil
}
func init() {
err := cache.Factories().Register("gcs", New)
if err != nil {
panic(err)
}
}
package gcs
import (
"encoding/json"
"fmt"
"io/ioutil"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type credentialsResolver interface {
Credentials() *common.CacheGCSCredentials
Resolve() error
}
const TypeServiceAccount = "service_account"
type credentialsFile struct {
Type string `json:"type"`
ClientEmail string `json:"client_email"`
PrivateKey string `json:"private_key"`
}
type defaultCredentialsResolver struct {
config *common.CacheGCSConfig
credentials *common.CacheGCSCredentials
}
func (cr *defaultCredentialsResolver) Credentials() *common.CacheGCSCredentials {
return cr.credentials
}
func (cr *defaultCredentialsResolver) Resolve() error {
if cr.config.CredentialsFile != "" {
return cr.readCredentialsFromFile()
}
return cr.readCredentialsFromConfig()
}
func (cr *defaultCredentialsResolver) readCredentialsFromFile() error {
data, err := ioutil.ReadFile(cr.config.CredentialsFile)
if err != nil {
return fmt.Errorf("error while reading credentials file: %w", err)
}
var credentialsFileContent credentialsFile
err = json.Unmarshal(data, &credentialsFileContent)
if err != nil {
return fmt.Errorf("error while parsing credentials file: %w", err)
}
if credentialsFileContent.Type != TypeServiceAccount {
return fmt.Errorf("unsupported credentials file type: %s", credentialsFileContent.Type)
}
logrus.Debugln("Credentials loaded from file. Skipping direct settings from Runner configuration file")
cr.credentials.AccessID = credentialsFileContent.ClientEmail
cr.credentials.PrivateKey = credentialsFileContent.PrivateKey
return nil
}
func (cr *defaultCredentialsResolver) readCredentialsFromConfig() error {
if cr.config.AccessID == "" || cr.config.PrivateKey == "" {
return fmt.Errorf("GCS config present, but credentials are not configured")
}
cr.credentials.AccessID = cr.config.AccessID
cr.credentials.PrivateKey = cr.config.PrivateKey
return nil
}
func newDefaultCredentialsResolver(config *common.CacheGCSConfig) (*defaultCredentialsResolver, error) {
if config == nil {
return nil, fmt.Errorf("config can't be nil")
}
credentials := &defaultCredentialsResolver{
config: config,
credentials: &common.CacheGCSCredentials{},
}
return credentials, nil
}
var credentialsResolverInitializer = newDefaultCredentialsResolver
package s3
import (
"fmt"
"net/http"
"net/url"
"time"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/cache"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type s3Adapter struct {
timeout time.Duration
config *common.CacheS3Config
objectName string
client minioClient
}
func (a *s3Adapter) GetDownloadURL() *url.URL {
URL, err := a.client.PresignedGetObject(a.config.BucketName, a.objectName, a.timeout, nil)
if err != nil {
logrus.WithError(err).Error("error while generating S3 pre-signed URL")
return nil
}
return URL
}
func (a *s3Adapter) GetUploadURL() *url.URL {
URL, err := a.client.PresignedPutObject(a.config.BucketName, a.objectName, a.timeout)
if err != nil {
logrus.WithError(err).Error("error while generating S3 pre-signed URL")
return nil
}
return URL
}
func (a *s3Adapter) GetUploadHeaders() http.Header {
return nil
}
func (a *s3Adapter) GetGoCloudURL() *url.URL {
return nil
}
func (a *s3Adapter) GetUploadEnv() map[string]string {
return nil
}
func New(config *common.CacheConfig, timeout time.Duration, objectName string) (cache.Adapter, error) {
s3 := config.S3
if s3 == nil {
return nil, fmt.Errorf("missing S3 configuration")
}
client, err := newMinioClient(s3)
if err != nil {
return nil, fmt.Errorf("error while creating S3 cache storage client: %w", err)
}
a := &s3Adapter{
config: s3,
timeout: timeout,
objectName: objectName,
client: client,
}
return a, nil
}
func init() {
err := cache.Factories().Register("s3", New)
if err != nil {
panic(err)
}
}
package s3
import (
"bytes"
"encoding/xml"
"io/ioutil"
"net/http"
)
type bucketLocationTripper struct {
bucketLocation string
}
// The Minio Golang library always attempts to query the bucket location and
// currently has no way of statically setting that value. To avoid that
// lookup, the Runner cache uses the library only to generate the URLs,
// forgoing the library's API for uploading and downloading files. The custom
// Roundtripper stubs out any network requests that would normally be made via
// the library.
func (b *bucketLocationTripper) RoundTrip(req *http.Request) (res *http.Response, err error) {
var buffer bytes.Buffer
err = xml.NewEncoder(&buffer).Encode(b.bucketLocation)
if err != nil {
return
}
res = &http.Response{
StatusCode: http.StatusOK,
Body: ioutil.NopCloser(&buffer),
}
return
}
func (b *bucketLocationTripper) CancelRequest(req *http.Request) {
// Do nothing
}
package s3
import (
"fmt"
"gitlab.com/gitlab-org/gitlab-runner/cache"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type s3CredentialsAdapter struct {
config *common.CacheS3Config
}
func (a *s3CredentialsAdapter) GetCredentials() map[string]string {
credMap := make(map[string]string)
// For IAM instance profiles, Go Cloud will fetch the credentials with the AWS SDK.
if a.config.AccessKey == "" || a.config.SecretKey == "" {
return credMap
}
credMap["AWS_ACCESS_KEY_ID"] = a.config.AccessKey
credMap["AWS_SECRET_ACCESS_KEY"] = a.config.SecretKey
return credMap
}
func NewS3CredentialsAdapter(config *common.CacheConfig) (cache.CredentialsAdapter, error) {
s3 := config.S3
if s3 == nil {
return nil, fmt.Errorf("missing S3 configuration")
}
a := &s3CredentialsAdapter{
config: s3,
}
return a, nil
}
func init() {
err := cache.CredentialsFactories().Register("s3", NewS3CredentialsAdapter)
if err != nil {
panic(err)
}
}
package s3
import (
"net/url"
"time"
"github.com/minio/minio-go/v6"
"github.com/minio/minio-go/v6/pkg/credentials"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
const DefaultAWSS3Server = "s3.amazonaws.com"
type minioClient interface {
PresignedGetObject(
bucketName string,
objectName string,
expires time.Duration,
reqParams url.Values,
) (*url.URL, error)
PresignedPutObject(bucketName string, objectName string, expires time.Duration) (*url.URL, error)
}
var newMinio = minio.New
var newMinioWithCredentials = minio.NewWithCredentials
var newMinioClient = func(s3 *common.CacheS3Config) (minioClient, error) {
var client *minio.Client
var err error
if s3.ShouldUseIAMCredentials() {
iam := credentials.NewIAM("")
client, err = newMinioWithCredentials(DefaultAWSS3Server, iam, true, "")
} else {
client, err = newMinio(s3.ServerAddress, s3.AccessKey, s3.SecretKey, !s3.Insecure)
}
if err != nil {
return nil, err
}
client.SetCustomTransport(&bucketLocationTripper{
bucketLocation: s3.BucketLocation,
})
return client, nil
}
package test
import (
"net/http"
"net/url"
"time"
"gitlab.com/gitlab-org/gitlab-runner/cache"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type testAdapter struct {
objectName string
useGoCloud bool
}
func (t *testAdapter) GetDownloadURL() *url.URL {
return t.getURL("download")
}
func (t *testAdapter) GetUploadURL() *url.URL {
return t.getURL("upload")
}
func (t *testAdapter) GetUploadHeaders() http.Header {
headers := http.Header{}
headers.Set("header-1", "a value")
return headers
}
func (t *testAdapter) GetGoCloudURL() *url.URL {
if t.useGoCloud {
u, _ := url.Parse("gocloud://test")
return u
}
return nil
}
func (t *testAdapter) GetUploadEnv() map[string]string {
return map[string]string{
"FIRST_VAR": "123",
"SECOND_VAR": "456",
}
}
func (t *testAdapter) getURL(operation string) *url.URL {
return &url.URL{
Scheme: "test",
Host: operation,
Path: t.objectName,
}
}
func New(_ *common.CacheConfig, _ time.Duration, objectName string) (cache.Adapter, error) {
return &testAdapter{objectName: objectName}, nil
}
func NewGoCloudAdapter(_ *common.CacheConfig, _ time.Duration, objectName string) (cache.Adapter, error) {
return &testAdapter{objectName: objectName, useGoCloud: true}, nil
}
func init() {
if err := cache.Factories().Register("test", New); err != nil {
panic(err)
}
if err := cache.Factories().Register("goCloudTest", NewGoCloudAdapter); err != nil {
panic(err)
}
}
package commands
import (
"fmt"
"net/http"
"strings"
"sync"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
"gitlab.com/gitlab-org/gitlab-runner/session"
"github.com/prometheus/client_golang/prometheus"
)
var numBuildsDesc = prometheus.NewDesc(
"gitlab_runner_jobs",
"The current number of running builds.",
[]string{"runner", "state", "stage", "executor_stage"},
nil,
)
var requestConcurrencyDesc = prometheus.NewDesc(
"gitlab_runner_request_concurrency",
"The current number of concurrent requests for a new job",
[]string{"runner"},
nil,
)
var requestConcurrencyExceededDesc = prometheus.NewDesc(
"gitlab_runner_request_concurrency_exceeded_total",
"Counter tracking exceeding of request concurrency",
[]string{"runner"},
nil,
)
type statePermutation struct {
runner string
buildState common.BuildRuntimeState
buildStage common.BuildStage
executorStage common.ExecutorStage
}
func newStatePermutationFromBuild(build *common.Build) statePermutation {
return statePermutation{
runner: build.Runner.ShortDescription(),
buildState: build.CurrentState(),
buildStage: build.CurrentStage(),
executorStage: build.CurrentExecutorStage(),
}
}
type runnerCounter struct {
builds int
requests int
requestConcurrencyExceeded int
}
type buildsHelper struct {
counters map[string]*runnerCounter
builds []*common.Build
lock sync.Mutex
jobsTotal *prometheus.CounterVec
jobDurationHistogram *prometheus.HistogramVec
}
func (b *buildsHelper) getRunnerCounter(runner *common.RunnerConfig) *runnerCounter {
if b.counters == nil {
b.counters = make(map[string]*runnerCounter)
}
counter := b.counters[runner.Token]
if counter == nil {
counter = &runnerCounter{}
b.counters[runner.Token] = counter
}
return counter
}
func (b *buildsHelper) findSessionByURL(url string) *session.Session {
b.lock.Lock()
defer b.lock.Unlock()
for _, build := range b.builds {
if strings.HasPrefix(url, build.Session.Endpoint+"/") {
return build.Session
}
}
return nil
}
func (b *buildsHelper) acquireBuild(runner *common.RunnerConfig) bool {
b.lock.Lock()
defer b.lock.Unlock()
counter := b.getRunnerCounter(runner)
if runner.Limit > 0 && counter.builds >= runner.Limit {
// Too many builds
return false
}
counter.builds++
return true
}
func (b *buildsHelper) releaseBuild(runner *common.RunnerConfig) bool {
b.lock.Lock()
defer b.lock.Unlock()
counter := b.getRunnerCounter(runner)
if counter.builds > 0 {
counter.builds--
return true
}
return false
}
func (b *buildsHelper) acquireRequest(runner *common.RunnerConfig) bool {
b.lock.Lock()
defer b.lock.Unlock()
counter := b.getRunnerCounter(runner)
if counter.requests >= runner.GetRequestConcurrency() {
counter.requestConcurrencyExceeded++
return false
}
counter.requests++
return true
}
func (b *buildsHelper) releaseRequest(runner *common.RunnerConfig) bool {
b.lock.Lock()
defer b.lock.Unlock()
counter := b.getRunnerCounter(runner)
if counter.requests > 0 {
counter.requests--
return true
}
return false
}
func (b *buildsHelper) addBuild(build *common.Build) {
if build == nil {
return
}
b.lock.Lock()
defer b.lock.Unlock()
runners := make(map[int]bool)
projectRunners := make(map[int]bool)
for _, otherBuild := range b.builds {
if otherBuild.Runner.Token != build.Runner.Token {
continue
}
runners[otherBuild.RunnerID] = true
if otherBuild.JobInfo.ProjectID != build.JobInfo.ProjectID {
continue
}
projectRunners[otherBuild.ProjectRunnerID] = true
}
for {
if !runners[build.RunnerID] {
break
}
build.RunnerID++
}
for {
if !projectRunners[build.ProjectRunnerID] {
break
}
build.ProjectRunnerID++
}
b.builds = append(b.builds, build)
b.jobsTotal.WithLabelValues(build.Runner.ShortDescription()).Inc()
}
func (b *buildsHelper) removeBuild(deleteBuild *common.Build) bool {
b.lock.Lock()
defer b.lock.Unlock()
b.jobDurationHistogram.
WithLabelValues(deleteBuild.Runner.ShortDescription()).
Observe(deleteBuild.Duration().Seconds())
for idx, build := range b.builds {
if build == deleteBuild {
b.builds = append(b.builds[0:idx], b.builds[idx+1:]...)
return true
}
}
return false
}
func (b *buildsHelper) buildsCount() int {
b.lock.Lock()
defer b.lock.Unlock()
return len(b.builds)
}
func (b *buildsHelper) statesAndStages() map[statePermutation]int {
b.lock.Lock()
defer b.lock.Unlock()
data := make(map[statePermutation]int)
for _, build := range b.builds {
state := newStatePermutationFromBuild(build)
data[state]++
}
return data
}
func (b *buildsHelper) runnersCounters() map[string]*runnerCounter {
b.lock.Lock()
defer b.lock.Unlock()
data := make(map[string]*runnerCounter)
for token, counter := range b.counters {
data[helpers.ShortenToken(token)] = counter
}
return data
}
// Describe implements prometheus.Collector.
func (b *buildsHelper) Describe(ch chan<- *prometheus.Desc) {
ch <- numBuildsDesc
ch <- requestConcurrencyDesc
ch <- requestConcurrencyExceededDesc
b.jobsTotal.Describe(ch)
b.jobDurationHistogram.Describe(ch)
}
// Collect implements prometheus.Collector.
func (b *buildsHelper) Collect(ch chan<- prometheus.Metric) {
builds := b.statesAndStages()
for state, count := range builds {
ch <- prometheus.MustNewConstMetric(
numBuildsDesc,
prometheus.GaugeValue,
float64(count),
state.runner,
string(state.buildState),
string(state.buildStage),
string(state.executorStage),
)
}
counters := b.runnersCounters()
for runner, counter := range counters {
ch <- prometheus.MustNewConstMetric(
requestConcurrencyDesc,
prometheus.GaugeValue,
float64(counter.requests),
runner,
)
ch <- prometheus.MustNewConstMetric(
requestConcurrencyExceededDesc,
prometheus.CounterValue,
float64(counter.requestConcurrencyExceeded),
runner,
)
}
b.jobsTotal.Collect(ch)
b.jobDurationHistogram.Collect(ch)
}
func (b *buildsHelper) ListJobsHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Add("X-List-Version", "2")
w.Header().Add("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
for _, job := range b.builds {
_, _ = fmt.Fprintf(
w,
"url=%s state=%s stage=%s executor_stage=%s duration=%s\n",
job.JobURL(),
job.CurrentState(),
job.CurrentStage(),
job.CurrentExecutorStage(),
job.Duration(),
)
}
}
func newBuildsHelper() buildsHelper {
return buildsHelper{
jobsTotal: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "gitlab_runner_jobs_total",
Help: "Total number of handled jobs",
},
[]string{"runner"},
),
jobDurationHistogram: prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "gitlab_runner_job_duration_seconds",
Help: "Histogram of job durations",
Buckets: []float64{30, 60, 300, 600, 1800, 3600, 7200, 10800, 18000, 36000},
},
[]string{"runner"},
),
}
}
package commands
import (
"fmt"
"net"
"os"
"path/filepath"
"strings"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/network"
)
func GetDefaultConfigFile() string {
return filepath.Join(getDefaultConfigDirectory(), "config.toml")
}
func getDefaultCertificateDirectory() string {
return filepath.Join(getDefaultConfigDirectory(), "certs")
}
type configOptions struct {
config *common.Config
ConfigFile string `short:"c" long:"config" env:"CONFIG_FILE" description:"Config file"`
}
func (c *configOptions) saveConfig() error {
return c.config.SaveConfig(c.ConfigFile)
}
func (c *configOptions) loadConfig() error {
config := common.NewConfig()
err := config.LoadConfig(c.ConfigFile)
if err != nil {
return err
}
c.config = config
return nil
}
func (c *configOptions) RunnerByName(name string) (*common.RunnerConfig, error) {
if c.config == nil {
return nil, fmt.Errorf("config has not been loaded")
}
for _, runner := range c.config.Runners {
if runner.Name == name {
return runner, nil
}
}
return nil, fmt.Errorf("could not find a runner with the name '%s'", name)
}
//nolint:lll
type configOptionsWithListenAddress struct {
configOptions
ListenAddress string `long:"listen-address" env:"LISTEN_ADDRESS" description:"Metrics / pprof server listening address"`
}
func (c *configOptionsWithListenAddress) listenAddress() (string, error) {
address := c.config.ListenAddress
if c.ListenAddress != "" {
address = c.ListenAddress
}
if address == "" {
return "", nil
}
_, port, err := net.SplitHostPort(address)
if err != nil && !strings.Contains(err.Error(), "missing port in address") {
return "", err
}
if port == "" {
return fmt.Sprintf("%s:%d", address, common.DefaultMetricsServerPort), nil
}
return address, nil
}
func init() {
configFile := os.Getenv("CONFIG_FILE")
if configFile == "" {
err := os.Setenv("CONFIG_FILE", GetDefaultConfigFile())
if err != nil {
logrus.WithError(err).Fatal("Couldn't set CONFIG_FILE environment variable")
}
}
network.CertificateDirectory = getDefaultCertificateDirectory()
}
// +build linux darwin freebsd openbsd
package commands
import (
"os"
"path/filepath"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
)
func getDefaultConfigDirectory() string {
if os.Getuid() == 0 {
return "/etc/gitlab-runner"
} else if homeDir := helpers.GetHomeDir(); homeDir != "" {
return filepath.Join(homeDir, ".gitlab-runner")
} else if currentDir := helpers.GetCurrentWorkingDirectory(); currentDir != "" {
return currentDir
}
panic("Cannot get default config file location")
}
package commands
import (
"os"
"os/exec"
"strings"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
clihelpers "gitlab.com/ayufan/golang-cli-helpers"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/gitlab_ci_yaml_parser"
// Force to load all executors, executes init() on them
_ "gitlab.com/gitlab-org/gitlab-runner/executors/custom"
_ "gitlab.com/gitlab-org/gitlab-runner/executors/docker"
_ "gitlab.com/gitlab-org/gitlab-runner/executors/parallels"
_ "gitlab.com/gitlab-org/gitlab-runner/executors/shell"
_ "gitlab.com/gitlab-org/gitlab-runner/executors/ssh"
_ "gitlab.com/gitlab-org/gitlab-runner/executors/virtualbox"
)
type ExecCommand struct {
common.RunnerSettings
Job string
Timeout int `long:"timeout" description:"Job execution timeout (in seconds)"`
}
// nolint:unparam
func (c *ExecCommand) runCommand(name string, arg ...string) (string, error) {
cmd := exec.Command(name, arg...)
cmd.Env = os.Environ()
cmd.Stderr = os.Stderr
result, err := cmd.Output()
return string(result), err
}
func (c *ExecCommand) createBuild(repoURL string, abortSignal chan os.Signal) (*common.Build, error) {
// Check if we have uncommitted changes
_, err := c.runCommand("git", "diff", "--quiet", "HEAD")
if err != nil {
logrus.Warningln("You most probably have uncommitted changes.")
logrus.Warningln("These changes will not be tested.")
}
// Parse Git settings
sha, err := c.runCommand("git", "rev-parse", "HEAD")
if err != nil {
return nil, err
}
beforeSha, err := c.runCommand("git", "rev-parse", "HEAD~1")
if err != nil {
beforeSha = "0000000000000000000000000000000000000000"
}
refName, err := c.runCommand("git", "rev-parse", "--abbrev-ref", "HEAD")
if err != nil {
return nil, err
}
jobResponse := common.JobResponse{
ID: 1,
Token: "",
AllowGitFetch: false,
JobInfo: common.JobInfo{
Name: "",
Stage: "",
ProjectID: 1,
ProjectName: "",
},
GitInfo: common.GitInfo{
RepoURL: repoURL,
Ref: strings.TrimSpace(refName),
Sha: strings.TrimSpace(sha),
BeforeSha: strings.TrimSpace(beforeSha),
},
RunnerInfo: common.RunnerInfo{
Timeout: c.getTimeout(),
},
}
runner := &common.RunnerConfig{
RunnerSettings: c.RunnerSettings,
}
return common.NewBuild(jobResponse, runner, abortSignal, nil)
}
func (c *ExecCommand) getTimeout() int {
if c.Timeout > 0 {
return c.Timeout
}
return common.DefaultExecTimeout
}
func (c *ExecCommand) Execute(context *cli.Context) {
wd, err := os.Getwd()
if err != nil {
logrus.Fatalln(err)
}
switch len(context.Args()) {
case 1:
c.Job = context.Args().Get(0)
default:
_ = cli.ShowSubcommandHelp(context)
os.Exit(1)
return
}
c.Executor = context.Command.Name
abortSignal := make(chan os.Signal)
doneSignal := make(chan int, 1)
go waitForInterrupts(nil, abortSignal, doneSignal, nil)
// Add self-volume to docker
if c.RunnerSettings.Docker == nil {
c.RunnerSettings.Docker = &common.DockerConfig{}
}
c.RunnerSettings.Docker.Volumes = append(c.RunnerSettings.Docker.Volumes, wd+":"+wd+":ro")
// Create build
build, err := c.createBuild(wd, abortSignal)
if err != nil {
logrus.Fatalln(err)
}
parser := gitlab_ci_yaml_parser.NewGitLabCiYamlParser(c.Job)
err = parser.ParseYaml(&build.JobResponse)
if err != nil {
logrus.Fatalln(err)
}
err = build.Run(&common.Config{}, &common.Trace{Writer: os.Stdout})
if err != nil {
logrus.Fatalln(err)
}
}
func init() {
cmd := &ExecCommand{}
flags := clihelpers.GetFlagsFromStruct(cmd)
cliCmd := cli.Command{
Name: "exec",
Usage: "execute a build locally",
}
for _, executorName := range common.GetExecutorNames() {
subCmd := cli.Command{
Name: executorName,
Usage: "use " + executorName + " executor",
Action: cmd.Execute,
Flags: flags,
}
cliCmd.Subcommands = append(cliCmd.Subcommands, subCmd)
}
common.RegisterCommand(cliCmd)
}
package commands
import (
"sync"
"time"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type healthData struct {
failures int
lastCheck time.Time
}
type healthHelper struct {
healthy map[string]*healthData
healthyLock sync.Mutex
}
func (mr *healthHelper) getHealth(id string) *healthData {
if mr.healthy == nil {
mr.healthy = map[string]*healthData{}
}
health := mr.healthy[id]
if health == nil {
health = &healthData{
lastCheck: time.Now(),
}
mr.healthy[id] = health
}
return health
}
func (mr *healthHelper) isHealthy(id string) bool {
mr.healthyLock.Lock()
defer mr.healthyLock.Unlock()
health := mr.getHealth(id)
if health.failures < common.HealthyChecks {
return true
}
if time.Since(health.lastCheck) > common.HealthCheckInterval*time.Second {
logrus.Errorln("Runner", id, "is not healthy, but will be checked!")
health.failures = 0
health.lastCheck = time.Now()
return true
}
return false
}
func (mr *healthHelper) makeHealthy(id string, healthy bool) {
mr.healthyLock.Lock()
defer mr.healthyLock.Unlock()
health := mr.getHealth(id)
if healthy {
health.failures = 0
health.lastCheck = time.Now()
} else {
health.failures++
if health.failures >= common.HealthyChecks {
logrus.Errorln("Runner", id, "is not healthy and will be disabled!")
}
}
}
package archive
import (
"context"
"errors"
"fmt"
"io"
"os"
)
var (
// ErrUnsupportedArchiveFormat is returned if an archiver or extractor format
// requested has not been registered.
ErrUnsupportedArchiveFormat = errors.New("unsupported archive format")
)
// CompressionLevel type for specifying a compression level.
type CompressionLevel int
// Compression levels from fastest (low/zero compression ratio) to slowest
// (high compression ratio).
const (
FastestCompression CompressionLevel = -2
FastCompression CompressionLevel = -1
DefaultCompression CompressionLevel = 0
SlowCompression CompressionLevel = 1
SlowestCompression CompressionLevel = 2
)
// Format type for specifying format.
type Format string
// Formats supported by GitLab.
const (
Raw Format = "raw"
Gzip Format = "gzip"
Zip Format = "zip"
)
var (
archivers = make(map[Format]NewArchiverFunc)
extractors = make(map[Format]NewExtractorFunc)
)
// Archiver is an interface for the Archive method.
type Archiver interface {
Archive(ctx context.Context, files map[string]os.FileInfo) error
}
// Extractor is an interface for the Extract method.
type Extractor interface {
Extract(ctx context.Context) error
}
// NewArchiverFunc is a function that can be registered (with Register()) and
// used to instantiate a new archiver (with NewArchiver()).
type NewArchiverFunc func(w io.Writer, dir string, level CompressionLevel) (Archiver, error)
// NewExtractorFunc is a function that can be registered (with Register()) and
// used to instantiate a new extractor (with NewExtractor()).
type NewExtractorFunc func(r io.ReaderAt, size int64, dir string) (Extractor, error)
// Register registers a new archiver, overriding the archiver and/or extractor
// for the format provided.
func Register(format Format, archiver NewArchiverFunc, extractor NewExtractorFunc) {
if archiver != nil {
archivers[format] = archiver
}
if extractor != nil {
extractors[format] = extractor
}
}
// NewArchiver returns a new Archiver of the specified format.
//
// The archiver will ensure that files to be archived are children of the
// directory provided.
func NewArchiver(format Format, w io.Writer, dir string, level CompressionLevel) (Archiver, error) {
fn := archivers[format]
if fn == nil {
return nil, fmt.Errorf("%q format: %w", format, ErrUnsupportedArchiveFormat)
}
return fn(w, dir, level)
}
// NewExtractor returns a new Extractor of the specified format.
//
// The extractor will extract files to the directory provided.
func NewExtractor(format Format, r io.ReaderAt, size int64, dir string) (Extractor, error) {
fn := extractors[format]
if fn == nil {
return nil, fmt.Errorf("%q format: %w", format, ErrUnsupportedArchiveFormat)
}
return fn(r, size, dir)
}
package fastzip
import (
"archive/zip"
"context"
"fmt"
"io"
"io/ioutil"
"os"
"github.com/saracen/fastzip"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
)
var flateLevels = map[archive.CompressionLevel]int{
archive.FastestCompression: 0,
archive.FastCompression: 1,
archive.DefaultCompression: 5,
archive.SlowCompression: 7,
archive.SlowestCompression: 9,
}
// archiver is a zip stream archiver.
type archiver struct {
w io.Writer
dir string
level archive.CompressionLevel
}
// NewArchiver returns a new Zip Archiver.
func NewArchiver(w io.Writer, dir string, level archive.CompressionLevel) (archive.Archiver, error) {
return &archiver{
w: w,
dir: dir,
level: level,
}, nil
}
// Archive archives all files provided.
func (a *archiver) Archive(ctx context.Context, files map[string]os.FileInfo) error {
tmpDir, err := ioutil.TempDir("", "fastzip")
if err != nil {
return fmt.Errorf("fastzip archiver unable to create temporary directory: %w", err)
}
defer os.RemoveAll(tmpDir)
opts := []fastzip.ArchiverOption{
fastzip.WithStageDirectory(tmpDir),
}
if a.level == archive.FastestCompression {
opts = append(opts, fastzip.WithArchiverMethod(zip.Store))
}
fa, err := fastzip.NewArchiver(a.w, a.dir, opts...)
if err != nil {
return err
}
if a.level != archive.FastestCompression {
fa.RegisterCompressor(zip.Deflate, fastzip.FlateCompressor(flateLevels[a.level]))
}
err = fa.Archive(ctx, files)
if cerr := fa.Close(); err == nil && cerr != nil {
return cerr
}
return err
}
package fastzip
import (
"context"
"io"
"github.com/saracen/fastzip"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
)
// extractor is a zip stream extractor.
type extractor struct {
r io.ReaderAt
size int64
dir string
}
// NewExtractor returns a new Zip Extractor.
func NewExtractor(r io.ReaderAt, size int64, dir string) (archive.Extractor, error) {
return &extractor{r: r, size: size, dir: dir}, nil
}
// Extract extracts files from the reader to the directory passed to
// NewExtractor.
func (e *extractor) Extract(ctx context.Context) error {
extractor, err := fastzip.NewExtractorFromReader(e.r, e.size, e.dir)
if err != nil {
return err
}
defer extractor.Close()
return extractor.Extract(ctx)
}
package gziplegacy
import (
"context"
"io"
"os"
"sort"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
)
func init() {
archive.Register(archive.Gzip, NewArchiver, nil)
}
// archiver is a gzip stream archiver.
type archiver struct {
w io.Writer
dir string
}
// NewArchiver returns a new Gzip Archiver.
func NewArchiver(w io.Writer, dir string, level archive.CompressionLevel) (archive.Archiver, error) {
return &archiver{w: w, dir: dir}, nil
}
// Archive archives all files as new gzip streams.
func (a *archiver) Archive(ctx context.Context, files map[string]os.FileInfo) error {
sorted := make([]string, 0, len(files))
for filename := range files {
sorted = append(sorted, filename)
}
sort.Strings(sorted)
return archives.CreateGzipArchive(a.w, sorted)
}
package raw
import (
"context"
"errors"
"io"
"os"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
)
func init() {
archive.Register(archive.Raw, NewArchiver, nil)
}
// ErrTooManyRawFiles is returned if more than one file is passed to the
// RawArchiver.
var ErrTooManyRawFiles = errors.New("only one file can be sent as raw")
// archiver is a raw archiver. It doesn't support compression nor multiple
// files.
type archiver struct {
w io.Writer
dir string
}
// NewArchiver returns a new Raw Archiver.
func NewArchiver(w io.Writer, dir string, level archive.CompressionLevel) (archive.Archiver, error) {
return &archiver{w: w, dir: dir}, nil
}
// Archive opens and copies a single file to the writer passed to
// NewRawArchiver. If more than one file is passed, ErrTooManyRawFiles is
// returned.
func (a *archiver) Archive(ctx context.Context, files map[string]os.FileInfo) error {
if len(files) > 1 {
return ErrTooManyRawFiles
}
for pathname := range files {
f, err := os.Open(pathname)
if err != nil {
return err
}
defer f.Close()
_, err = io.Copy(a.w, f)
return err
}
return nil
}
package ziplegacy
import (
"context"
"io"
"os"
"sort"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
)
func init() {
archive.Register(archive.Zip, NewArchiver, NewExtractor)
}
// archiver is a zip stream archiver.
type archiver struct {
w io.Writer
dir string
}
// NewArchiver returns a new Zip Archiver.
func NewArchiver(w io.Writer, dir string, level archive.CompressionLevel) (archive.Archiver, error) {
return &archiver{w: w, dir: dir}, nil
}
// Archive archives all files as new gzip streams.
func (a *archiver) Archive(ctx context.Context, files map[string]os.FileInfo) error {
sorted := make([]string, 0, len(files))
for filename := range files {
sorted = append(sorted, filename)
}
sort.Strings(sorted)
return archives.CreateZipArchive(a.w, sorted)
}
package ziplegacy
import (
"archive/zip"
"context"
"io"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
"gitlab.com/gitlab-org/gitlab-runner/helpers/archives"
)
// extractor is a zip stream extractor.
type extractor struct {
r io.ReaderAt
size int64
dir string
}
// NewExtractor returns a new Zip Extractor.
func NewExtractor(r io.ReaderAt, size int64, dir string) (archive.Extractor, error) {
return &extractor{r: r, size: size, dir: dir}, nil
}
// Extract extracts files from the reader to the directory passed to
// NewZipExtractor.
func (e *extractor) Extract(ctx context.Context) error {
zr, err := zip.NewReader(e.r, e.size)
if err != nil {
return err
}
return archives.ExtractZipArchive(zr)
}
package helpers
import (
"os"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive/fastzip"
"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
// auto-register default archivers/extractors
_ "gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive/gziplegacy"
_ "gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive/raw"
_ "gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive/ziplegacy"
"github.com/sirupsen/logrus"
)
func init() {
// enable fastzip archiver/extractor
logger := logrus.WithField("name", featureflags.UseFastzip)
if on := featureflags.IsOn(logger, os.Getenv(featureflags.UseFastzip)); on {
archive.Register(archive.Zip, fastzip.NewArchiver, fastzip.NewExtractor)
}
}
// GetCompressionLevel converts the compression level name to compression level type
// https://docs.gitlab.com/ee/ci/runners/README.html#artifact-and-cache-settings
func GetCompressionLevel(name string) archive.CompressionLevel {
switch name {
case "fastest":
return archive.FastestCompression
case "fast":
return archive.FastCompression
case "slow":
return archive.SlowCompression
case "slowest":
return archive.SlowestCompression
case "default", "":
return archive.DefaultCompression
}
logrus.Warningf("compression level %q is invalid, falling back to default", name)
return archive.DefaultCompression
}
package helpers
import (
"context"
"fmt"
"io/ioutil"
"os"
"time"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/meter"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/log"
"gitlab.com/gitlab-org/gitlab-runner/network"
)
//nolint:lll
type ArtifactsDownloaderCommand struct {
common.JobCredentials
retryHelper
network common.Network
meter.TransferMeterCommand
DirectDownload bool `long:"direct-download" env:"FF_USE_DIRECT_DOWNLOAD" description:"Support direct download for data stored externally to GitLab"`
}
func (c *ArtifactsDownloaderCommand) directDownloadFlag(retry int) *bool {
// We want to send `?direct_download=true`
// Use direct download only on a first attempt
if c.DirectDownload && retry == 0 {
return &c.DirectDownload
}
// We don't want to send `?direct_download=false`
return nil
}
func (c *ArtifactsDownloaderCommand) download(file string, retry int) error {
artifactsFile, err := os.Create(file)
if err != nil {
return fmt.Errorf("creating target file: %w", err)
}
// Close() is checked properly inside of DownloadArtifacts() call
defer func() { _ = artifactsFile.Close() }()
writer := meter.NewWriter(
artifactsFile,
c.TransferMeterFrequency,
meter.LabelledRateFormat(os.Stdout, "Downloading artifacts", meter.UnknownTotalSize),
)
// Close() is checked properly inside of DownloadArtifacts() call
defer func() { _ = writer.Close() }()
switch c.network.DownloadArtifacts(c.JobCredentials, writer, c.directDownloadFlag(retry)) {
case common.DownloadSucceeded:
return nil
case common.DownloadNotFound:
return os.ErrNotExist
case common.DownloadForbidden:
return os.ErrPermission
case common.DownloadFailed:
return retryableErr{err: os.ErrInvalid}
default:
return os.ErrInvalid
}
}
func (c *ArtifactsDownloaderCommand) Execute(cliContext *cli.Context) {
log.SetRunnerFormatter()
wd, err := os.Getwd()
if err != nil {
logrus.Fatalln("Unable to get working directory")
}
if c.URL == "" || c.Token == "" {
logrus.Fatalln("Missing runner credentials")
}
if c.ID <= 0 {
logrus.Fatalln("Missing build ID")
}
// Create temporary file
file, err := ioutil.TempFile("", "artifacts")
if err != nil {
logrus.Fatalln(err)
}
_ = file.Close()
defer func() { _ = os.Remove(file.Name()) }()
// Download artifacts file
err = c.doRetry(func(retry int) error {
return c.download(file.Name(), retry)
})
if err != nil {
logrus.Fatalln(err)
}
f, size, err := openZip(file.Name())
if err != nil {
logrus.Fatalln(err)
}
defer f.Close()
extractor, err := archive.NewExtractor(archive.Zip, f, size, wd)
if err != nil {
logrus.Fatalln(err)
}
// Extract artifacts file
err = extractor.Extract(context.Background())
if err != nil {
logrus.Fatalln(err)
}
}
func openZip(filename string) (*os.File, int64, error) {
f, err := os.Open(filename)
if err != nil {
return nil, 0, err
}
fi, err := f.Stat()
if err != nil {
f.Close()
return nil, 0, err
}
return f, fi.Size(), nil
}
func init() {
common.RegisterCommand2(
"artifacts-downloader",
"download and extract build artifacts (internal)",
&ArtifactsDownloaderCommand{
network: network.NewGitLabClient(),
retryHelper: retryHelper{
Retry: 2,
RetryTime: time.Second,
},
},
)
}
package helpers
import (
"context"
"errors"
"io"
"os"
"path/filepath"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/meter"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/retry"
"gitlab.com/gitlab-org/gitlab-runner/log"
"gitlab.com/gitlab-org/gitlab-runner/network"
)
const (
DefaultUploadName = "default"
defaultTries = 3
serviceUnavailableTries = 6
)
var (
errServiceUnavailable = errors.New("service unavailable")
errTooLarge = errors.New("too large")
)
//nolint:lll
type ArtifactsUploaderCommand struct {
common.JobCredentials
fileArchiver
meter.TransferMeterCommand
network common.Network
Name string `long:"name" description:"The name of the archive"`
ExpireIn string `long:"expire-in" description:"When to expire artifacts"`
Format common.ArtifactFormat `long:"artifact-format" description:"Format of generated artifacts"`
Type string `long:"artifact-type" description:"Type of generated artifacts"`
CompressionLevel string `long:"compression-level" env:"ARTIFACT_COMPRESSION_LEVEL" description:"Compression level (fastest, fast, default, slow, slowest)"`
}
func (c *ArtifactsUploaderCommand) artifactFilename(name string, format common.ArtifactFormat) string {
name = filepath.Base(name)
if name == "" || name == "." {
name = DefaultUploadName
}
switch format {
case common.ArtifactFormatZip:
return name + ".zip"
case common.ArtifactFormatGzip:
return name + ".gz"
}
return name
}
func (c *ArtifactsUploaderCommand) createReadStream() (string, io.ReadCloser, error) {
if len(c.files) == 0 {
return "", nil, nil
}
format := c.Format
if format == common.ArtifactFormatDefault {
format = common.ArtifactFormatZip
}
filename := c.artifactFilename(c.Name, format)
pr, pw := io.Pipe()
archiver, err := archive.NewArchiver(archive.Format(format), pw, c.wd, GetCompressionLevel(c.CompressionLevel))
if err != nil {
_ = pr.CloseWithError(err)
return filename, nil, err
}
go func() {
err := archiver.Archive(context.Background(), c.files)
_ = pw.CloseWithError(err)
}()
return filename, pr, nil
}
func (c *ArtifactsUploaderCommand) Run() error {
artifactsName, stream, err := c.createReadStream()
if err != nil {
return err
}
if stream == nil {
logrus.Errorln("No files to upload")
return nil
}
defer func() { _ = stream.Close() }()
// Create the archive
options := common.ArtifactsOptions{
BaseName: artifactsName,
ExpireIn: c.ExpireIn,
Format: c.Format,
Type: c.Type,
}
stream = meter.NewReader(
stream,
c.TransferMeterFrequency,
meter.LabelledRateFormat(os.Stdout, "Uploading artifacts", meter.UnknownTotalSize),
)
// Upload the data
switch c.network.UploadRawArtifacts(c.JobCredentials, stream, options) {
case common.UploadSucceeded:
return nil
case common.UploadForbidden:
return os.ErrPermission
case common.UploadTooLarge:
return errTooLarge
case common.UploadFailed:
return retryableErr{err: os.ErrInvalid}
case common.UploadServiceUnavailable:
return retryableErr{err: errServiceUnavailable}
default:
return os.ErrInvalid
}
}
func (c *ArtifactsUploaderCommand) ShouldRetry(tries int, err error) bool {
var errAs retryableErr
if !errors.As(err, &errAs) {
return false
}
maxTries := defaultTries
if errors.Is(errAs, errServiceUnavailable) {
maxTries = serviceUnavailableTries
}
if tries >= maxTries {
return false
}
return true
}
func (c *ArtifactsUploaderCommand) Execute(*cli.Context) {
log.SetRunnerFormatter()
if c.URL == "" || c.Token == "" {
logrus.Fatalln("Missing runner credentials")
}
if c.ID <= 0 {
logrus.Fatalln("Missing build ID")
}
// Enumerate files
err := c.enumerate()
if err != nil {
logrus.Fatalln(err)
}
// If the upload fails, exit with a non-zero exit code to indicate an issue?
logger := logrus.WithField("context", "artifacts-uploader")
retryable := retry.New(retry.WithLogrus(c, logger))
err = retryable.Run()
if err != nil {
logrus.Fatalln(err)
}
}
func init() {
common.RegisterCommand2(
"artifacts-uploader",
"create and upload build artifacts (internal)",
&ArtifactsUploaderCommand{
network: network.NewGitLabClient(),
Name: "artifacts",
},
)
}
package helpers
import (
"context"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/meter"
"gitlab.com/gitlab-org/gitlab-runner/common"
url_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/url"
"gitlab.com/gitlab-org/gitlab-runner/log"
"gocloud.dev/blob"
_ "gocloud.dev/blob/azureblob" // Needed to register the Azure driver
)
//nolint:lll
type CacheArchiverCommand struct {
fileArchiver
retryHelper
meter.TransferMeterCommand
File string `long:"file" description:"The path to file"`
URL string `long:"url" description:"URL of remote cache resource (pre-signed URL)"`
GoCloudURL string `long:"gocloud-url" description:"Go Cloud URL of remote cache resource (requires credentials)"`
Timeout int `long:"timeout" description:"Overall timeout for cache uploading request (in minutes)"`
Headers []string `long:"header" description:"HTTP headers to send with PUT request (in form of 'key:value')"`
CompressionLevel string `long:"compression-level" env:"CACHE_COMPRESSION_LEVEL" description:"Compression level (fastest, fast, default, slow, slowest)"`
client *CacheClient
mux *blob.URLMux
}
func (c *CacheArchiverCommand) getClient() *CacheClient {
if c.client == nil {
c.client = NewCacheClient(c.Timeout)
}
return c.client
}
func (c *CacheArchiverCommand) upload(_ int) error {
file, err := os.Open(c.File)
if err != nil {
return err
}
defer func() { _ = file.Close() }()
fi, err := file.Stat()
if err != nil {
return err
}
rc := meter.NewReader(
file,
c.TransferMeterFrequency,
meter.LabelledRateFormat(os.Stdout, "Uploading cache", fi.Size()),
)
defer rc.Close()
if c.GoCloudURL != "" {
return c.handleGoCloudURL(rc)
}
return c.handlePresignedURL(fi, rc)
}
func (c *CacheArchiverCommand) handlePresignedURL(fi os.FileInfo, file io.Reader) error {
logrus.Infoln("Uploading", filepath.Base(c.File), "to", url_helpers.CleanURL(c.URL))
req, err := http.NewRequest(http.MethodPut, c.URL, file)
if err != nil {
return retryableErr{err: err}
}
c.setHeaders(req, fi)
req.ContentLength = fi.Size()
resp, err := c.getClient().Do(req)
if err != nil {
return retryableErr{err: err}
}
defer func() { _ = resp.Body.Close() }()
return retryOnServerError(resp)
}
func (c *CacheArchiverCommand) handleGoCloudURL(file io.Reader) error {
logrus.Infoln("Uploading", filepath.Base(c.File), "to", url_helpers.CleanURL(c.GoCloudURL))
if c.mux == nil {
c.mux = blob.DefaultURLMux()
}
ctx, cancelWrite := context.WithCancel(context.Background())
defer cancelWrite()
u, err := url.Parse(c.GoCloudURL)
if err != nil {
return err
}
objectName := strings.TrimLeft(u.Path, "/")
if objectName == "" {
return fmt.Errorf("no object name provided")
}
b, err := c.mux.OpenBucket(ctx, c.GoCloudURL)
if err != nil {
return err
}
defer b.Close()
writer, err := b.NewWriter(ctx, objectName, nil)
if err != nil {
return err
}
if _, err = io.Copy(writer, file); err != nil {
cancelWrite()
if writerErr := writer.Close(); writerErr != nil {
logrus.WithError(writerErr).Error("error closing Go cloud upload after copy failure")
}
return err
}
if err := writer.Close(); err != nil {
return err
}
return nil
}
func (c *CacheArchiverCommand) createZipFile(filename string) error {
err := os.MkdirAll(filepath.Dir(filename), 0700)
if err != nil {
return err
}
f, err := ioutil.TempFile(filepath.Dir(filename), "archive_")
if err != nil {
return err
}
defer os.Remove(f.Name())
defer f.Close()
logrus.Debugln("Temporary file:", f.Name())
archiver, err := archive.NewArchiver(archive.Zip, f, c.wd, GetCompressionLevel(c.CompressionLevel))
if err != nil {
return err
}
// Create archive
err = archiver.Archive(context.Background(), c.files)
if err != nil {
return err
}
err = f.Close()
if err != nil {
return err
}
return os.Rename(f.Name(), filename)
}
func (c *CacheArchiverCommand) Execute(*cli.Context) {
log.SetRunnerFormatter()
if c.File == "" {
logrus.Fatalln("Missing --file")
}
// Enumerate files
err := c.enumerate()
if err != nil {
logrus.Fatalln(err)
}
// Check if list of files changed
if !c.isFileChanged(c.File) {
logrus.Infoln("Archive is up to date!")
return
}
// Create archive
err = c.createZipFile(c.File)
if err != nil {
logrus.Fatalln(err)
}
// Upload archive if needed
if c.URL != "" || c.GoCloudURL != "" {
err := c.doRetry(c.upload)
if err != nil {
logrus.Fatalln(err)
}
} else {
logrus.Infoln(
"No URL provided, cache will be not uploaded to shared cache server. " +
"Cache will be stored only locally.")
}
}
func (c *CacheArchiverCommand) setHeaders(req *http.Request, fi os.FileInfo) {
if len(c.Headers) > 0 {
for _, header := range c.Headers {
parsed := strings.SplitN(header, ":", 2)
if len(parsed) != 2 {
continue
}
req.Header.Set(strings.TrimSpace(parsed[0]), strings.TrimSpace(parsed[1]))
}
return
}
// Set default headers
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))
}
func init() {
common.RegisterCommand2(
"cache-archiver",
"create and upload cache artifacts (internal)",
&CacheArchiverCommand{
retryHelper: retryHelper{
Retry: 2,
RetryTime: time.Second,
},
},
)
}
package helpers
import (
"net"
"net/http"
"time"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type CacheClient struct {
http.Client
}
func (c *CacheClient) prepareClient(timeout int) {
if timeout > 0 {
c.Timeout = time.Duration(timeout) * time.Minute
} else {
c.Timeout = time.Duration(common.DefaultCacheRequestTimeout) * time.Minute
}
}
func (c *CacheClient) prepareTransport() {
c.Transport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 10 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
DisableCompression: true,
}
}
func NewCacheClient(timeout int) *CacheClient {
client := &CacheClient{}
client.prepareClient(timeout)
client.prepareTransport()
return client
}
package helpers
import (
"context"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strconv"
"time"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/archive"
"gitlab.com/gitlab-org/gitlab-runner/commands/helpers/meter"
"gitlab.com/gitlab-org/gitlab-runner/common"
url_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/url"
"gitlab.com/gitlab-org/gitlab-runner/log"
)
type CacheExtractorCommand struct {
retryHelper
meter.TransferMeterCommand
File string `long:"file" description:"The file containing your cache artifacts"`
URL string `long:"url" description:"URL of remote cache resource"`
Timeout int `long:"timeout" description:"Overall timeout for cache downloading request (in minutes)"`
client *CacheClient
}
func (c *CacheExtractorCommand) getClient() *CacheClient {
if c.client == nil {
c.client = NewCacheClient(c.Timeout)
}
return c.client
}
func checkIfUpToDate(path string, resp *http.Response) (bool, time.Time) {
fi, _ := os.Lstat(path)
date, _ := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified"))
return fi != nil && !date.After(fi.ModTime()), date
}
func getRemoteCacheSize(resp *http.Response) int64 {
length, _ := strconv.Atoi(resp.Header.Get("Content-Length"))
if length <= 0 {
return meter.UnknownTotalSize
}
return int64(length)
}
func (c *CacheExtractorCommand) download(_ int) error {
err := os.MkdirAll(filepath.Dir(c.File), 0700)
if err != nil {
return err
}
resp, err := c.getCache()
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }()
upToDate, date := checkIfUpToDate(c.File, resp)
if upToDate {
logrus.Infoln(filepath.Base(c.File), "is up to date")
return nil
}
file, err := ioutil.TempFile(filepath.Dir(c.File), "cache")
if err != nil {
return err
}
defer func() {
_ = file.Close()
_ = os.Remove(file.Name())
}()
logrus.Infoln("Downloading", filepath.Base(c.File), "from", url_helpers.CleanURL(c.URL))
writer := meter.NewWriter(
file,
c.TransferMeterFrequency,
meter.LabelledRateFormat(os.Stdout, "Downloading cache", getRemoteCacheSize(resp)),
)
// Close() is checked properly bellow, where the file handling is being finalized
defer func() { _ = writer.Close() }()
_, err = io.Copy(writer, resp.Body)
if err != nil {
return retryableErr{err: err}
}
err = os.Chtimes(file.Name(), time.Now(), date)
if err != nil {
return err
}
err = writer.Close()
if err != nil {
return err
}
err = os.Rename(file.Name(), c.File)
if err != nil {
return err
}
return nil
}
func (c *CacheExtractorCommand) getCache() (*http.Response, error) {
resp, err := c.getClient().Get(c.URL)
if err != nil {
return nil, retryableErr{err: err}
}
if resp.StatusCode == http.StatusNotFound {
_ = resp.Body.Close()
return nil, os.ErrNotExist
}
return resp, retryOnServerError(resp)
}
func (c *CacheExtractorCommand) Execute(cliContext *cli.Context) {
log.SetRunnerFormatter()
wd, err := os.Getwd()
if err != nil {
logrus.Fatalln("Unable to get working directory")
}
if c.File == "" {
logrus.Fatalln("Missing cache file")
}
if c.URL != "" {
err := c.doRetry(c.download)
if err != nil {
logrus.Fatalln(err)
}
} else {
logrus.Infoln(
"No URL provided, cache will not be downloaded from shared cache server. " +
"Instead a local version of cache will be extracted.")
}
f, size, err := openZip(c.File)
if os.IsNotExist(err) {
return
}
if err != nil {
logrus.Fatalln(err)
}
defer f.Close()
extractor, err := archive.NewExtractor(archive.Zip, f, size, wd)
if err != nil {
logrus.Fatalln(err)
}
err = extractor.Extract(context.Background())
if err != nil {
logrus.Fatalln(err)
}
}
func init() {
common.RegisterCommand2(
"cache-extractor",
"download and extract cache artifacts (internal)",
&CacheExtractorCommand{
retryHelper: retryHelper{
Retry: 2,
RetryTime: time.Second,
},
},
)
}
package helpers
import (
"os"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
// CacheInitCommand will take a single directory/file path and initialize it
// correctly for it to be used for cache. This command tries to support spaces
// in directories name by using the the flags to specify which entries you want
// to initialize.
type CacheInitCommand struct{}
func (c *CacheInitCommand) Execute(ctx *cli.Context) {
if ctx.NArg() == 0 {
logrus.Fatal("No arguments passed, at least 1 path is required.")
}
for _, path := range ctx.Args() {
err := os.Chmod(path, os.ModePerm)
if err != nil {
logrus.WithError(err).Error("failed to chmod path")
}
}
}
func init() {
common.RegisterCommand2("cache-init", "changed permissions for cache paths (internal)", &CacheInitCommand{})
}
package helpers
import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"time"
"github.com/bmatcuk/doublestar"
"github.com/sirupsen/logrus"
)
type fileArchiver struct {
Paths []string `long:"path" description:"Add paths to archive"`
Exclude []string `long:"exclude" description:"Exclude paths from the archive"`
Untracked bool `long:"untracked" description:"Add git untracked files"`
Verbose bool `long:"verbose" description:"Detailed information"`
wd string
files map[string]os.FileInfo
excluded map[string]int64
}
func (c *fileArchiver) isChanged(modTime time.Time) bool {
for _, info := range c.files {
if modTime.Before(info.ModTime()) {
return true
}
}
return false
}
func (c *fileArchiver) isFileChanged(fileName string) bool {
ai, err := os.Stat(fileName)
if ai != nil {
if !c.isChanged(ai.ModTime()) {
return false
}
} else if !os.IsNotExist(err) {
logrus.Warningln(err)
}
return true
}
func (c *fileArchiver) sortedFiles() []string {
files := make([]string, len(c.files))
i := 0
for file := range c.files {
files[i] = file
i++
}
sort.Strings(files)
return files
}
func (c *fileArchiver) process(match string) bool {
var absolute, relative string
var err error
absolute, err = filepath.Abs(match)
if err == nil {
// Let's try to find a real relative path to an absolute from working directory
relative, err = filepath.Rel(c.wd, absolute)
}
if err == nil {
// Process path only if it lives in our build directory
if !strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
excluded, rule := c.isExcluded(relative)
if excluded {
c.exclude(rule)
return false
}
err = c.add(relative)
} else {
err = errors.New("not supported: outside build directory")
}
}
if err == nil {
return true
}
if os.IsNotExist(err) {
// We hide the error that file doesn't exist
return false
}
logrus.Warningf("%s: %v", match, err)
return false
}
func (c *fileArchiver) isExcluded(path string) (bool, string) {
for _, pattern := range c.Exclude {
excluded, err := doublestar.PathMatch(pattern, path)
if err == nil && excluded {
return true, pattern
}
}
return false, ""
}
func (c *fileArchiver) exclude(rule string) {
c.excluded[rule]++
}
func (c *fileArchiver) add(path string) error {
// Always use slashes
path = filepath.ToSlash(path)
// Check if file exist
info, err := os.Lstat(path)
if err == nil {
c.files[path] = info
}
return err
}
func (c *fileArchiver) processPaths() {
for _, path := range c.Paths {
matches, err := doublestar.Glob(path)
if err != nil {
logrus.Warningf("%s: %v", path, err)
continue
}
found := 0
for _, match := range matches {
err := filepath.Walk(match, func(path string, info os.FileInfo, err error) error {
if c.process(path) {
found++
}
return nil
})
if err != nil {
logrus.Warningln("Walking", match, err)
}
}
if found == 0 {
logrus.Warningf("%s: no matching files", path)
} else {
logrus.Infof("%s: found %d matching files and directories", path, found)
}
}
}
func (c *fileArchiver) processUntracked() {
if !c.Untracked {
return
}
found := 0
var output bytes.Buffer
cmd := exec.Command("git", "ls-files", "-o", "-z")
cmd.Env = os.Environ()
cmd.Stdout = &output
cmd.Stderr = os.Stderr
logrus.Debugln("Executing command:", strings.Join(cmd.Args, " "))
err := cmd.Run()
if err != nil {
logrus.Warningf("untracked: %v", err)
return
}
reader := bufio.NewReader(&output)
for {
line, err := reader.ReadString(0)
if err == io.EOF {
break
} else if err != nil {
logrus.Warningln(err)
break
}
if c.process(line[:len(line)-1]) {
found++
}
}
if found == 0 {
logrus.Warningf("untracked: no files")
} else {
logrus.Infof("untracked: found %d files", found)
}
}
func (c *fileArchiver) enumerate() error {
wd, err := os.Getwd()
if err != nil {
return fmt.Errorf("failed to get current working directory: %w", err)
}
c.wd = wd
c.files = make(map[string]os.FileInfo)
c.excluded = make(map[string]int64)
c.processPaths()
c.processUntracked()
for path, count := range c.excluded {
logrus.Infof("%s: excluded %d files", path, count)
}
return nil
}
package helpers
import (
"fmt"
"net"
"os"
"strings"
"time"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type HealthCheckCommand struct{}
func (c *HealthCheckCommand) Execute(ctx *cli.Context) {
var addr, port string
for _, e := range os.Environ() {
parts := strings.Split(e, "=")
switch {
case len(parts) != 2:
continue
case strings.HasSuffix(parts[0], "_TCP_ADDR"):
addr = parts[1]
case strings.HasSuffix(parts[0], "_TCP_PORT"):
port = parts[1]
}
}
if addr == "" || port == "" {
logrus.Fatalln("No HOST or PORT found")
}
_, _ = fmt.Fprintf(os.Stdout, "waiting for TCP connection to %s:%s...", addr, port)
for {
conn, err := net.Dial("tcp", net.JoinHostPort(addr, port))
if err != nil {
time.Sleep(time.Second)
continue
}
_ = conn.Close()
return
}
}
func init() {
common.RegisterCommand2("health-check", "check health for a specific address", &HealthCheckCommand{})
}
package meter
import (
"fmt"
"io"
"math"
"time"
)
func FormatByteRate(b uint64, d time.Duration) string {
b = uint64(float64(b) / math.Max(time.Nanosecond.Seconds(), d.Seconds()))
rate, prefix := formatBytes(b)
if prefix == 0 {
return fmt.Sprintf("%d B/s", int(rate))
}
return fmt.Sprintf("%.1f %cB/s", rate, prefix)
}
func FormatBytes(b uint64) string {
size, prefix := formatBytes(b)
if prefix == 0 {
return fmt.Sprintf("%d B", int(size))
}
return fmt.Sprintf("%.2f %cB", size, prefix)
}
func formatBytes(b uint64) (float64, byte) {
const (
unit = 1000
prefix = "KMGTPE"
)
if b < unit {
return float64(b), 0
}
div := int64(unit)
exp := 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return float64(b) / float64(div), prefix[exp]
}
func LabelledRateFormat(w io.Writer, label string, totalSize int64) UpdateCallback {
return func(written uint64, since time.Duration, done bool) {
known := ""
if totalSize > UnknownTotalSize {
known = "/" + FormatBytes(uint64(totalSize))
}
line := fmt.Sprintf(
"\r%s %s%s (%s) ",
label,
FormatBytes(written),
known,
FormatByteRate(written, since),
)
if done {
_, _ = fmt.Fprintln(w, line)
return
}
_, _ = fmt.Fprint(w, line)
}
}
package meter
import (
"sync"
"sync/atomic"
"time"
)
const UnknownTotalSize = 0
type TransferMeterCommand struct {
//nolint:lll
TransferMeterFrequency time.Duration `long:"transfer-meter-frequency" env:"TRANSFER_METER_FREQUENCY" description:"If set to more than 0s it enables an interactive transfer meter"`
}
type UpdateCallback func(written uint64, since time.Duration, done bool)
type meter struct {
count uint64
done, notify chan struct{}
close sync.Once
}
func newMeter() *meter {
return &meter{
done: make(chan struct{}),
notify: make(chan struct{}),
}
}
func (m *meter) start(frequency time.Duration, fn UpdateCallback) {
if frequency < time.Second {
frequency = time.Second
}
started := time.Now()
go func() {
defer close(m.done)
ticker := time.NewTicker(frequency)
defer ticker.Stop()
for {
fn(atomic.LoadUint64(&m.count), time.Since(started), false)
select {
case <-ticker.C:
case <-m.notify:
fn(atomic.LoadUint64(&m.count), time.Since(started), true)
return
}
}
}()
}
func (m *meter) doClose() {
m.close.Do(func() {
// notify we're done
close(m.notify)
// wait for close
<-m.done
})
}
package meter
import (
"io"
"sync/atomic"
"time"
)
type reader struct {
*meter
r io.ReadCloser
}
func NewReader(r io.ReadCloser, frequency time.Duration, fn UpdateCallback) io.ReadCloser {
if frequency == 0 {
return r
}
m := &reader{
r: r,
meter: newMeter(),
}
m.start(frequency, fn)
return m
}
func (m *reader) Read(p []byte) (int, error) {
n, err := m.r.Read(p)
atomic.AddUint64(&m.count, uint64(n))
return n, err
}
func (m *reader) Close() error {
m.doClose()
return m.r.Close()
}
package meter
import (
"io"
"sync/atomic"
"time"
)
type writer struct {
*meter
w io.WriteCloser
}
func NewWriter(w io.WriteCloser, frequency time.Duration, fn UpdateCallback) io.WriteCloser {
if frequency == 0 {
return w
}
m := &writer{
w: w,
meter: newMeter(),
}
m.start(frequency, fn)
return m
}
func (m *writer) Write(p []byte) (int, error) {
n, err := m.w.Write(p)
atomic.AddUint64(&m.count, uint64(n))
return n, err
}
func (m *writer) Close() error {
m.doClose()
return m.w.Close()
}
package helpers
import (
"bufio"
"errors"
"fmt"
"io"
"os"
"time"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
const (
// defaultReaderBufferSize is the size of the line buffer.
// Docker/Kubernetes use the same size to split lines
defaultReaderBufferSize = 16 * 1024
defaultCheckFileExistsInterval = time.Second
pollFileContentsTimeout = 500 * time.Millisecond
outputLogFileNotExistsExitCode = 100
)
var (
errWaitingFileTimeout = errors.New("timeout waiting for file to be created")
errNoAttemptsToOpenFile = errors.New("no attempts to open log file configured")
)
type logStreamProvider interface {
Open() (readSeekCloser, error)
}
type readSeekCloser interface {
io.ReadSeeker
io.Closer
}
// checkedFile checks whether a file exists when the underlying
// File's Read method returns io.EOF. If a file is deleted from
// the outside the Go file descriptor isn't invalidated and we
// keep getting io.EOF oblivious to the fact that the file
// no longer exists
type checkedFile struct {
*os.File
}
func (c *checkedFile) Read(p []byte) (int, error) {
n, err := c.File.Read(p)
if errors.Is(err, io.EOF) {
_, statErr := os.Stat(c.File.Name())
if os.IsNotExist(statErr) {
err = statErr
}
}
return n, err
}
type fileLogStreamProvider struct {
waitFileTimeout time.Duration
path string
}
func (p *fileLogStreamProvider) Open() (readSeekCloser, error) {
attempts := int(p.waitFileTimeout / defaultCheckFileExistsInterval)
if attempts < 1 {
return nil, errNoAttemptsToOpenFile
}
for i := 0; i < attempts; i++ {
f, err := os.Open(p.path)
if os.IsNotExist(err) {
time.Sleep(defaultCheckFileExistsInterval)
continue
}
return &checkedFile{File: f}, err
}
return nil, errWaitingFileTimeout
}
type logOutputWriter interface {
Write(string)
}
type streamLogOutputWriter struct {
stream io.Writer
}
func (s *streamLogOutputWriter) Write(data string) {
_, _ = fmt.Fprint(s.stream, data)
}
type ReadLogsCommand struct {
Path string `long:"path"`
Offset int64 `long:"offset"`
WaitFileTimeout time.Duration `long:"wait-file-timeout"`
logStreamProvider logStreamProvider
logOutputWriter logOutputWriter
readerBufferSize int
}
func newReadLogsCommand() *ReadLogsCommand {
return &ReadLogsCommand{
logOutputWriter: &streamLogOutputWriter{stream: os.Stdout},
readerBufferSize: defaultReaderBufferSize,
// by default check if the file exists at least once
WaitFileTimeout: defaultCheckFileExistsInterval,
}
}
func (c *ReadLogsCommand) Execute(*cli.Context) {
err := c.execute()
switch {
case os.IsNotExist(err):
os.Exit(outputLogFileNotExistsExitCode)
case err != nil:
c.logOutputWriter.Write(fmt.Sprintf("error reading logs %v\n", err))
os.Exit(1)
}
}
func (c *ReadLogsCommand) execute() error {
c.logStreamProvider = &fileLogStreamProvider{
waitFileTimeout: c.WaitFileTimeout,
path: c.Path,
}
return c.readLogs()
}
func (c *ReadLogsCommand) readLogs() error {
s, r, err := c.openFileReader()
if err != nil {
return err
}
defer s.Close()
offset := c.Offset
for {
buf, err := r.ReadSlice('\n')
if len(buf) > 0 {
offset += int64(len(buf))
// if the buffer was filled by a message larger than the
// buffer size we must make sure that it ends with a new line
// so it gets properly handled by the executor which splits by new lines
if buf[len(buf)-1] != '\n' {
buf = append(buf, '\n')
}
c.logOutputWriter.Write(fmt.Sprintf("%d %s", offset, buf))
}
// io.EOF means that we reached the end of the file
// we try reading from it again to see if there are new contents
// bufio.ErrBufferFull means that the message was larger than the buffer
// we print the message so far along with a new line character
// and continue reading the rest of it from the stream
if errors.Is(err, io.EOF) {
time.Sleep(pollFileContentsTimeout)
} else if err != nil && !errors.Is(err, bufio.ErrBufferFull) {
return err
}
}
}
func (c *ReadLogsCommand) openFileReader() (readSeekCloser, *bufio.Reader, error) {
s, err := c.logStreamProvider.Open()
if err != nil {
return nil, nil, err
}
_, err = s.Seek(c.Offset, 0)
if err != nil {
_ = s.Close()
return nil, nil, err
}
return s, bufio.NewReaderSize(s, c.readerBufferSize), nil
}
func init() {
common.RegisterCommand2(
"read-logs",
"reads job logs from a file, used by kubernetes executor (internal)",
newReadLogsCommand(),
)
}
package helpers
import (
"fmt"
"net/http"
"time"
"github.com/sirupsen/logrus"
)
type retryHelper struct {
Retry int `long:"retry" description:"How many times to retry upload"`
RetryTime time.Duration `long:"retry-time" description:"How long to wait between retries"`
}
// retryableErr indicates that an error can be retried. To specify that an error
// can be retried simply wrap the original error. For example:
//
// retryableErr{err: errors.New("some error")}
type retryableErr struct {
err error
}
func (e retryableErr) Unwrap() error {
return e.err
}
func (e retryableErr) Error() string {
return e.err.Error()
}
func (r *retryHelper) doRetry(handler func(int) error) error {
err := handler(0)
for retry := 1; retry <= r.Retry; retry++ {
if _, ok := err.(retryableErr); !ok {
return err
}
time.Sleep(r.RetryTime)
logrus.WithError(err).Warningln("Retrying...")
err = handler(retry)
}
return err
}
// retryOnServerError will take the response and check if the the error should
// be of type retryableErr or not. When the status code is of 5xx it will be a
// retryableErr.
func retryOnServerError(resp *http.Response) error {
if resp.StatusCode/100 == 2 {
return nil
}
_ = resp.Body.Close()
err := fmt.Errorf("received: %s", resp.Status)
if resp.StatusCode/100 == 5 {
err = retryableErr{err: err}
}
return err
}
package commands
import (
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type ListCommand struct {
configOptions
}
func (c *ListCommand) Execute(context *cli.Context) {
err := c.loadConfig()
if err != nil {
logrus.Warningln(err)
return
}
logrus.WithFields(logrus.Fields{
"ConfigFile": c.ConfigFile,
}).Println("Listing configured runners")
for _, runner := range c.config.Runners {
logrus.WithFields(logrus.Fields{
"Executor": runner.RunnerSettings.Executor,
"Token": runner.RunnerCredentials.Token,
"URL": runner.RunnerCredentials.URL,
}).Println(runner.Name)
}
}
func init() {
common.RegisterCommand2("list", "List all configured runners", &ListCommand{})
}
package commands
import (
"context"
"errors"
"fmt"
"net"
"net/http"
"net/http/pprof"
"os"
"os/signal"
"runtime"
"syscall"
"time"
"github.com/kardianos/service"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
"gitlab.com/gitlab-org/gitlab-runner/helpers/certificate"
prometheus_helper "gitlab.com/gitlab-org/gitlab-runner/helpers/prometheus"
"gitlab.com/gitlab-org/gitlab-runner/helpers/sentry"
service_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/service"
"gitlab.com/gitlab-org/gitlab-runner/log"
"gitlab.com/gitlab-org/gitlab-runner/network"
"gitlab.com/gitlab-org/gitlab-runner/session"
)
var (
concurrentDesc = prometheus.NewDesc(
"gitlab_runner_concurrent",
"The current value of concurrent setting",
nil,
nil,
)
limitDesc = prometheus.NewDesc(
"gitlab_runner_limit",
"The current value of concurrent setting",
[]string{"runner"},
nil,
)
)
type RunCommand struct {
configOptionsWithListenAddress
network common.Network
healthHelper
buildsHelper buildsHelper
ServiceName string `short:"n" long:"service" description:"Use different names for different services"`
WorkingDirectory string `short:"d" long:"working-directory" description:"Specify custom working directory"`
User string `short:"u" long:"user" description:"Use specific user to execute shell scripts"`
Syslog bool `long:"syslog" description:"Log to system service logger" env:"LOG_SYSLOG"`
sentryLogHook sentry.LogHook
prometheusLogHook prometheus_helper.LogHook
failuresCollector *prometheus_helper.FailuresCollector
networkRequestStatusesCollector prometheus.Collector
sessionServer *session.Server
// abortBuilds is used to abort running builds
abortBuilds chan os.Signal
// runInterruptSignal is used to abort current operation (scaling workers, waiting for config)
runInterruptSignal chan os.Signal
// reloadSignal is used to trigger forceful config reload
reloadSignal chan os.Signal
// stopSignals is to catch a signals notified to process: SIGTERM, SIGQUIT, Interrupt, Kill
stopSignals chan os.Signal
// stopSignal is used to preserve the signal that was used to stop the
// process In case this is SIGQUIT it makes to finish all builds and session
// server.
stopSignal os.Signal
// runFinished is used to notify that run() did finish
runFinished chan bool
currentWorkers int
}
func (mr *RunCommand) log() *logrus.Entry {
return logrus.WithField("builds", mr.buildsHelper.buildsCount())
}
// Start is the method implementing `github.com/kardianos/service`.`Interface`
// interface. It's responsible for a non-blocking initialization of the process. When it exits,
// the main control flow is passed to runWait() configured as service's RunWait method. Take a look
// into Execute() for details.
func (mr *RunCommand) Start(_ service.Service) error {
mr.abortBuilds = make(chan os.Signal)
mr.runInterruptSignal = make(chan os.Signal, 1)
mr.reloadSignal = make(chan os.Signal, 1)
mr.runFinished = make(chan bool, 1)
mr.stopSignals = make(chan os.Signal)
mr.log().Info("Starting multi-runner from ", mr.ConfigFile, "...")
userModeWarning(false)
if len(mr.WorkingDirectory) > 0 {
err := os.Chdir(mr.WorkingDirectory)
if err != nil {
return err
}
}
err := mr.loadConfig()
if err != nil {
return err
}
// Start should not block. Do the actual work async.
go mr.run()
return nil
}
func (mr *RunCommand) loadConfig() error {
err := mr.configOptions.loadConfig()
if err != nil {
return err
}
// Set log level
err = mr.updateLoggingConfiguration()
if err != nil {
return err
}
// pass user to execute scripts as specific user
if mr.User != "" {
mr.config.User = mr.User
}
mr.healthy = nil
mr.log().Println("Configuration loaded")
mr.log().Debugln(helpers.ToYAML(mr.config))
// initialize sentry
if mr.config.SentryDSN != nil {
var err error
mr.sentryLogHook, err = sentry.NewLogHook(*mr.config.SentryDSN)
if err != nil {
mr.log().WithError(err).Errorln("Sentry failure")
}
} else {
mr.sentryLogHook = sentry.LogHook{}
}
return nil
}
func (mr *RunCommand) updateLoggingConfiguration() error {
reloadNeeded := false
if mr.config.LogLevel != nil && !log.Configuration().IsLevelSetWithCli() {
err := log.Configuration().SetLevel(*mr.config.LogLevel)
if err != nil {
return err
}
reloadNeeded = true
}
if mr.config.LogFormat != nil && !log.Configuration().IsFormatSetWithCli() {
err := log.Configuration().SetFormat(*mr.config.LogFormat)
if err != nil {
return err
}
reloadNeeded = true
}
if reloadNeeded {
log.Configuration().ReloadConfiguration()
}
return nil
}
// run is the main method of RunCommand. It's started asynchronously by services support
// through `Start` method and is responsible for initializing all goroutines handling
// concurrent, multi-runner execution of jobs.
// When mr.stopSignal is broadcasted (after `Stop` is called by services support)
// this method waits for all workers to be terminated and closes the mr.runFinished
// channel, which is the signal that the command was properly terminated (this is the only
// valid, properly terminated exit flow for `gitlab-runner run`).
func (mr *RunCommand) run() {
mr.setupMetricsAndDebugServer()
mr.setupSessionServer()
runners := make(chan *common.RunnerConfig)
go mr.feedRunners(runners)
signal.Notify(mr.stopSignals, syscall.SIGQUIT, syscall.SIGTERM, os.Interrupt)
signal.Notify(mr.reloadSignal, syscall.SIGHUP)
startWorker := make(chan int)
stopWorker := make(chan bool)
go mr.startWorkers(startWorker, stopWorker, runners)
workerIndex := 0
// Update number of workers and reload configuration.
// Exits when mr.runInterruptSignal receives a signal.
for mr.stopSignal == nil {
signaled := mr.updateWorkers(&workerIndex, startWorker, stopWorker)
if signaled != nil {
break
}
signaled = mr.updateConfig()
if signaled != nil {
break
}
}
// Wait for workers to shutdown
for mr.currentWorkers > 0 {
stopWorker <- true
mr.currentWorkers--
}
mr.log().Info("All workers stopped. Can exit now")
close(mr.runFinished)
}
func (mr *RunCommand) setupMetricsAndDebugServer() {
listenAddress, err := mr.listenAddress()
if err != nil {
mr.log().Errorf("invalid listen address: %s", err.Error())
return
}
if listenAddress == "" {
mr.log().Info("listen_address not defined, metrics & debug endpoints disabled")
return
}
// We separate out the listener creation here so that we can return an error if
// the provided address is invalid or there is some other listener error.
listener, err := net.Listen("tcp", listenAddress)
if err != nil {
mr.log().WithError(err).Fatal("Failed to create listener for metrics server")
}
mux := http.NewServeMux()
go func() {
err := http.Serve(listener, mux)
if err != nil {
mr.log().WithError(err).Fatal("Metrics server terminated")
}
}()
mr.serveMetrics(mux)
mr.serveDebugData(mux)
mr.servePprof(mux)
mr.log().
WithField("address", listenAddress).
Info("Metrics server listening")
}
func (mr *RunCommand) serveMetrics(mux *http.ServeMux) {
registry := prometheus.NewRegistry()
// Metrics about the runner's business logic.
registry.MustRegister(&mr.buildsHelper)
registry.MustRegister(mr)
// Metrics about API connections
registry.MustRegister(mr.networkRequestStatusesCollector)
// Metrics about jobs failures
registry.MustRegister(mr.failuresCollector)
// Metrics about catched errors
registry.MustRegister(&mr.prometheusLogHook)
// Metrics about the program's build version.
registry.MustRegister(common.AppVersion.NewMetricsCollector())
// Go-specific metrics about the process (GC stats, goroutines, etc.).
registry.MustRegister(prometheus.NewGoCollector())
// Go-unrelated process metrics (memory usage, file descriptors, etc.).
registry.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
// Register all executor provider collectors
for _, provider := range common.GetExecutorProviders() {
if collector, ok := provider.(prometheus.Collector); ok && collector != nil {
registry.MustRegister(collector)
}
}
mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
}
func (mr *RunCommand) serveDebugData(mux *http.ServeMux) {
mux.HandleFunc("/debug/jobs/list", mr.buildsHelper.ListJobsHandler)
}
func (mr *RunCommand) servePprof(mux *http.ServeMux) {
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
}
func (mr *RunCommand) setupSessionServer() {
if mr.config.SessionServer.ListenAddress == "" {
mr.log().Info("[session_server].listen_address not defined, session endpoints disabled")
return
}
var err error
mr.sessionServer, err = session.NewServer(
session.ServerConfig{
AdvertiseAddress: mr.config.SessionServer.AdvertiseAddress,
ListenAddress: mr.config.SessionServer.ListenAddress,
ShutdownTimeout: common.ShutdownTimeout * time.Second,
},
mr.log(),
certificate.X509Generator{},
mr.buildsHelper.findSessionByURL,
)
if err != nil {
mr.log().WithError(err).Fatal("Failed to create session server")
}
go func() {
err := mr.sessionServer.Start()
if err != nil {
mr.log().WithError(err).Fatal("Session server terminated")
}
}()
mr.log().
WithField("address", mr.config.SessionServer.ListenAddress).
Info("Session server listening")
}
// feedRunners works until a stopSignal was saved.
// It is responsible for feeding the runners (workers) to channel, which
// asynchronously ends with job requests being made and jobs being executed
// by concurrent workers.
// This is also the place where check interval is calculated and
// applied.
func (mr *RunCommand) feedRunners(runners chan *common.RunnerConfig) {
for mr.stopSignal == nil {
mr.log().Debugln("Feeding runners to channel")
config := mr.config
// If no runners wait full interval to test again
if len(config.Runners) == 0 {
time.Sleep(config.GetCheckInterval())
continue
}
interval := config.GetCheckInterval() / time.Duration(len(config.Runners))
// Feed runner with waiting exact amount of time
for _, runner := range config.Runners {
mr.feedRunner(runner, runners)
time.Sleep(interval)
}
}
mr.log().
WithField("StopSignal", mr.stopSignal).
Debug("Stopping feeding runners to channel")
}
func (mr *RunCommand) feedRunner(runner *common.RunnerConfig, runners chan *common.RunnerConfig) {
if !mr.isHealthy(runner.UniqueID()) {
return
}
runners <- runner
}
// startWorkers is responsible for starting the workers (up to the number
// defined by `concurrent`) and assigning a runner processing method to them.
func (mr *RunCommand) startWorkers(startWorker chan int, stopWorker chan bool, runners chan *common.RunnerConfig) {
for mr.stopSignal == nil {
id := <-startWorker
go mr.processRunners(id, stopWorker, runners)
}
}
// processRunners is responsible for processing a Runner on a worker (when received
// a runner information sent to the channel by feedRunners) and for terminating the worker
// (when received an information on stoWorker chan - provided by updateWorkers)
func (mr *RunCommand) processRunners(id int, stopWorker chan bool, runners chan *common.RunnerConfig) {
mr.log().
WithField("worker", id).
Debugln("Starting worker")
for mr.stopSignal == nil {
select {
case runner := <-runners:
err := mr.processRunner(id, runner, runners)
if err != nil {
mr.log().
WithFields(logrus.Fields{
"runner": runner.ShortDescription(),
"executor": runner.Executor,
}).
WithError(err).
Warn("Failed to process runner")
}
// force GC cycle after processing build
runtime.GC()
case <-stopWorker:
mr.log().
WithField("worker", id).
Debugln("Stopping worker")
return
}
}
<-stopWorker
}
// processRunner is responsible for handling one job on a specified runner.
// First it acquires the Build to check if `limit` was met. If it's still in the capacity
// it creates the debug session (for debug terminal), triggers a job request to configured
// GitLab instance and finally creates and finishes the job.
// To speed-up jobs handling before starting the job this method "requeues" the runner to another
// worker (by feeding the channel normally handled by feedRunners).
func (mr *RunCommand) processRunner(
id int,
runner *common.RunnerConfig,
runners chan *common.RunnerConfig,
) (err error) {
provider := common.GetExecutorProvider(runner.Executor)
if provider == nil {
return
}
executorData, err := provider.Acquire(runner)
if err != nil {
return fmt.Errorf("failed to update executor: %w", err)
}
defer provider.Release(runner, executorData)
if !mr.buildsHelper.acquireBuild(runner) {
logrus.WithFields(logrus.Fields{
"runner": runner.ShortDescription(),
"worker": id,
}).Debug("Failed to request job, runner limit met")
return
}
defer mr.buildsHelper.releaseBuild(runner)
buildSession, sessionInfo, err := mr.createSession(provider)
if err != nil {
return
}
// Receive a new build
trace, jobData, err := mr.requestJob(runner, sessionInfo)
if err != nil || jobData == nil {
return
}
defer func() { mr.traceOutcome(trace, err) }()
// Create a new build
build, err := common.NewBuild(*jobData, runner, mr.abortBuilds, executorData)
if err != nil {
return
}
build.Session = buildSession
build.ArtifactUploader = mr.network.UploadRawArtifacts
// Add build to list of builds to assign numbers
mr.buildsHelper.addBuild(build)
defer mr.buildsHelper.removeBuild(build)
// Process the same runner by different worker again
// to speed up taking the builds
mr.requeueRunner(runner, runners)
// Process a build
return build.Run(mr.config, trace)
}
func (mr *RunCommand) traceOutcome(trace common.JobTrace, err error) {
if err != nil {
fmt.Fprintln(trace, err.Error())
trace.Fail(err, common.JobFailureData{Reason: common.RunnerSystemFailure})
} else {
trace.Success()
}
}
// createSession checks if debug server is supported by configured executor and if the
// debug server was configured. If both requirements are met, then it creates a debug session
// that will be assigned to newly created job.
func (mr *RunCommand) createSession(provider common.ExecutorProvider) (*session.Session, *common.SessionInfo, error) {
var features common.FeaturesInfo
if err := provider.GetFeatures(&features); err != nil {
return nil, nil, err
}
if mr.sessionServer == nil || !features.Session {
return nil, nil, nil
}
sess, err := session.NewSession(mr.log())
if err != nil {
return nil, nil, err
}
sessionInfo := &common.SessionInfo{
URL: mr.sessionServer.AdvertiseAddress + sess.Endpoint,
Certificate: string(mr.sessionServer.CertificatePublicKey),
Authorization: sess.Token,
}
return sess, sessionInfo, err
}
// requestJob will check if the runner can send another concurrent request to
// GitLab, if not the return value is nil.
func (mr *RunCommand) requestJob(
runner *common.RunnerConfig,
sessionInfo *common.SessionInfo,
) (common.JobTrace, *common.JobResponse, error) {
if !mr.buildsHelper.acquireRequest(runner) {
mr.log().WithField("runner", runner.ShortDescription()).
Debugln("Failed to request job: runner requestConcurrency meet")
return nil, nil, nil
}
defer mr.buildsHelper.releaseRequest(runner)
jobData, healthy := mr.doJobRequest(context.Background(), runner, sessionInfo)
mr.makeHealthy(runner.UniqueID(), healthy)
if jobData == nil {
return nil, nil, nil
}
// Make sure to always close output
jobCredentials := &common.JobCredentials{
ID: jobData.ID,
Token: jobData.Token,
}
trace, err := mr.network.ProcessJob(*runner, jobCredentials)
if err != nil {
jobInfo := common.UpdateJobInfo{
ID: jobCredentials.ID,
State: common.Failed,
FailureReason: common.RunnerSystemFailure,
}
// send failure once
mr.network.UpdateJob(*runner, jobCredentials, jobInfo)
return nil, nil, err
}
trace.SetFailuresCollector(mr.failuresCollector)
return trace, jobData, nil
}
// doJobRequest will execute the request for a new job, respecting an interruption
// caused by interrupt signals or process execution finalization
func (mr *RunCommand) doJobRequest(
ctx context.Context,
runner *common.RunnerConfig,
sessionInfo *common.SessionInfo,
) (*common.JobResponse, bool) {
// Terminate opened requests to GitLab when interrupt signal
// is broadcast.
ctx, cancelFn := context.WithCancel(ctx)
defer cancelFn()
go func() {
select {
case <-mr.runInterruptSignal:
cancelFn()
case <-mr.runFinished:
cancelFn()
case <-ctx.Done():
}
}()
return mr.network.RequestJob(ctx, *runner, sessionInfo)
}
// requeueRunner feeds the runners channel in a non-blocking way. This replicates the
// behavior of feedRunners and speeds-up jobs handling. But if the channel is full, the
// method just exits without blocking.
func (mr *RunCommand) requeueRunner(runner *common.RunnerConfig, runners chan *common.RunnerConfig) {
runnerLog := mr.log().WithField("runner", runner.ShortDescription())
select {
case runners <- runner:
runnerLog.Debugln("Requeued the runner")
default:
runnerLog.Debugln("Failed to requeue the runner")
}
}
// updateWorkers, called periodically from run() is responsible for scaling the pool
// of workers. By worker we don't understand a `[[runners]]` entry, but a "slot" that will
// use one of the runners to request and handle a job.
// The size of the workers pool is controlled by `concurrent` setting. This method is responsible
// for the fact that `concurrent` defines the upper number of jobs that can be concurrently handled
// by GitLab Runner process.
func (mr *RunCommand) updateWorkers(workerIndex *int, startWorker chan int, stopWorker chan bool) os.Signal {
concurrentLimit := mr.config.Concurrent
if concurrentLimit < 1 {
mr.log().Fatalln("Concurrent is less than 1 - no jobs will be processed")
}
for mr.currentWorkers > concurrentLimit {
// Too many workers. Trigger stop on one of them
// or exit if termination signal was broadcasted.
select {
case stopWorker <- true:
case signaled := <-mr.runInterruptSignal:
return signaled
}
mr.currentWorkers--
}
for mr.currentWorkers < concurrentLimit {
// Too few workers. Trigger a creation of a new one
// or exit if termination signal was broadcasted.
select {
case startWorker <- *workerIndex:
case signaled := <-mr.runInterruptSignal:
return signaled
}
mr.currentWorkers++
*workerIndex++
}
return nil
}
func (mr *RunCommand) updateConfig() os.Signal {
select {
case <-time.After(common.ReloadConfigInterval * time.Second):
err := mr.checkConfig()
if err != nil {
mr.log().Errorln("Failed to load config", err)
}
case <-mr.reloadSignal:
err := mr.loadConfig()
if err != nil {
mr.log().Errorln("Failed to load config", err)
}
case signaled := <-mr.runInterruptSignal:
return signaled
}
return nil
}
func (mr *RunCommand) checkConfig() (err error) {
info, err := os.Stat(mr.ConfigFile)
if err != nil {
return err
}
if !mr.config.ModTime.Before(info.ModTime()) {
return nil
}
err = mr.loadConfig()
if err != nil {
mr.log().Errorln("Failed to load config", err)
// don't reload the same file
mr.config.ModTime = info.ModTime()
return
}
return nil
}
// Stop is the method implementing `github.com/kardianos/service`.`Interface`
// interface. It's responsible for triggering the process stop.
// First it starts a goroutine that starts broadcasting the interrupt signal (used to stop
// workers scaling goroutine).
// Next it triggers graceful shutdown, which will be handled only if a proper signal is used.
// At the end it triggers the forceful shutdown, which handles the forceful the process termination.
func (mr *RunCommand) Stop(_ service.Service) error {
if mr.stopSignal == nil {
mr.stopSignal = os.Interrupt
}
go mr.interruptRun()
defer func() {
if mr.sessionServer != nil {
mr.sessionServer.Close()
}
}()
// On Windows, we convert SIGTERM and SIGINT signals into a SIGQUIT.
//
// This enforces *graceful* termination on the first signal received, and a forceful shutdown
// on the second.
//
// This slightly differs from other operating systems. On other systems, receiving a SIGQUIT
// works the same way (gracefully) but receiving a SIGTERM and SIGQUIT always results
// in an immediate forceful shutdown.
//
// This handling has to be different as SIGQUIT is not a signal the os/signal package translates
// any Windows control concepts to.
if runtime.GOOS == "windows" {
mr.stopSignal = syscall.SIGQUIT
}
err := mr.handleGracefulShutdown()
if err == nil {
return nil
}
mr.log().
WithError(err).
Warning("Graceful shutdown not finished properly")
err = mr.handleForcefulShutdown()
if err == nil {
return nil
}
mr.log().
WithError(err).
Warning("Forceful shutdown not finished properly")
return err
}
// interruptRun broadcasts interrupt signal, which exits the workers
// scaling goroutine.
func (mr *RunCommand) interruptRun() {
mr.log().Debug("Broadcasting interrupt signal")
// Pump interrupt signal
for {
mr.runInterruptSignal <- mr.stopSignal
}
}
// handleGracefulShutdown is responsible for handling the "graceful" strategy of exiting.
// It's executed only when specific signal is used to terminate the process.
// At this moment feedRunners() should exit and workers scaling is being terminated.
// This means that new jobs will be not requested. handleGracefulShutdown() will ensure that
// the process will not exit until `mr.runFinished` is closed, so all jobs were finished and
// all workers terminated. It may however exit if another signal - other than the gracefulShutdown
// signal - is received.
func (mr *RunCommand) handleGracefulShutdown() error {
// We wait till we have a SIGQUIT
for mr.stopSignal == syscall.SIGQUIT {
mr.log().
WithField("StopSignal", mr.stopSignal).
Warning("Starting graceful shutdown, waiting for builds to finish")
// Wait for other signals to finish builds
select {
case mr.stopSignal = <-mr.stopSignals:
// We received a new signal
mr.log().WithField("stop-signal", mr.stopSignal).Warning("[handleGracefulShutdown] received stop signal")
case <-mr.runFinished:
// Everything finished we can exit now
return nil
}
}
return fmt.Errorf("received stop signal: %v", mr.stopSignal)
}
// handleForcefulShutdown is executed if handleGracefulShutdown exited with an error
// (which means that a signal forcing shutdown was used instead of the signal
// specific for graceful shutdown).
// It calls mr.abortAllBuilds which will broadcast abort signal which finally
// ends with jobs termination.
// Next it waits for one of the following events:
// 1. Another signal was sent to process, which is handled as force exit and
// triggers exit of the method and finally process termination without
// waiting for anything else.
// 2. ShutdownTimeout is exceeded. If waiting for shutdown will take more than
// defined time, the process will be forceful terminated just like in the
// case when second signal is sent.
// 3. mr.runFinished was closed, which means that all termination was done
// properly.
//
// After this method exits, Stop returns it error and finally the
// `github.com/kardianos/service` service mechanism will finish
// process execution.
func (mr *RunCommand) handleForcefulShutdown() error {
mr.log().
WithField("StopSignal", mr.stopSignal).
Warning("Starting forceful shutdown")
go mr.abortAllBuilds()
// Wait for graceful shutdown or abort after timeout
for {
select {
case mr.stopSignal = <-mr.stopSignals:
mr.log().WithField("stop-signal", mr.stopSignal).Warning("[handleForcefulShutdown] received stop signal")
return fmt.Errorf("forced exit with stop signal: %v", mr.stopSignal)
case <-time.After(common.ShutdownTimeout * time.Second):
return errors.New("shutdown timed out")
case <-mr.runFinished:
// Everything finished we can exit now
return nil
}
}
}
// abortAllBuilds broadcasts abort signal, which ends with all currently executed
// jobs being interrupted and terminated.
func (mr *RunCommand) abortAllBuilds() {
mr.log().Debug("Broadcasting job abort signal")
// Pump signal to abort all current builds
for {
mr.abortBuilds <- mr.stopSignal
}
}
func (mr *RunCommand) Execute(_ *cli.Context) {
svcConfig := &service.Config{
Name: mr.ServiceName,
DisplayName: mr.ServiceName,
Description: defaultDescription,
Arguments: []string{"run"},
Option: service.KeyValue{
"RunWait": mr.runWait,
},
}
svc, err := service_helpers.New(mr, svcConfig)
if err != nil {
logrus.WithError(err).
Fatalln("Service creation failed")
}
if mr.Syslog {
log.SetSystemLogger(logrus.StandardLogger(), svc)
}
logrus.AddHook(&mr.sentryLogHook)
logrus.AddHook(&mr.prometheusLogHook)
err = svc.Run()
if err != nil {
logrus.WithError(err).
Fatal("Service run failed")
}
}
// runWait is the blocking mechanism for `github.com/kardianos/service`
// service. It's started after Start exited and should block the control flow. When it exits,
// then the Stop is executed and service shutdown should be handled.
// For Runner it waits for the stopSignal to be received by the process. When it will happen,
// it's saved in mr.stopSignal and runWait() exits, triggering the shutdown handling.
func (mr *RunCommand) runWait() {
mr.log().Debugln("Waiting for stop signal")
// Save the stop signal and exit to execute Stop()
stopSignal := <-mr.stopSignals
mr.stopSignal = stopSignal
mr.log().WithField("stop-signal", stopSignal).Warning("[runWait] received stop signal")
}
// Describe implements prometheus.Collector.
func (mr *RunCommand) Describe(ch chan<- *prometheus.Desc) {
ch <- concurrentDesc
ch <- limitDesc
}
// Collect implements prometheus.Collector.
func (mr *RunCommand) Collect(ch chan<- prometheus.Metric) {
config := mr.config
ch <- prometheus.MustNewConstMetric(
concurrentDesc,
prometheus.GaugeValue,
float64(config.Concurrent),
)
for _, runner := range config.Runners {
ch <- prometheus.MustNewConstMetric(
limitDesc,
prometheus.GaugeValue,
float64(runner.Limit),
runner.ShortDescription(),
)
}
}
func init() {
requestStatusesCollector := network.NewAPIRequestStatusesMap()
common.RegisterCommand2(
"run",
"run multi runner service",
&RunCommand{
ServiceName: defaultServiceName,
network: network.NewGitLabClientWithRequestStatusesMap(requestStatusesCollector),
networkRequestStatusesCollector: requestStatusesCollector,
prometheusLogHook: prometheus_helper.NewLogHook(),
failuresCollector: prometheus_helper.NewFailuresCollector(),
buildsHelper: newBuildsHelper(),
},
)
}
package commands
import (
"bufio"
"fmt"
"os"
"os/signal"
"runtime"
"strings"
"github.com/imdario/mergo"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/ssh"
"gitlab.com/gitlab-org/gitlab-runner/network"
"gitlab.com/gitlab-org/gitlab-runner/shells"
)
type configTemplate struct {
*common.Config
ConfigFile string `long:"config" env:"TEMPLATE_CONFIG_FILE" description:"Path to the configuration template file"`
}
func (c *configTemplate) Enabled() bool {
return c.ConfigFile != ""
}
func (c *configTemplate) MergeTo(config *common.RunnerConfig) error {
err := c.loadConfigTemplate()
if err != nil {
return errors.Wrap(err, "couldn't load configuration template file")
}
if len(c.Runners) != 1 {
return errors.New("configuration template must contain exactly one [[runners]] entry")
}
err = mergo.Merge(config, c.Runners[0])
if err != nil {
return errors.Wrap(err, "error while merging configuration with configuration template")
}
return nil
}
func (c *configTemplate) loadConfigTemplate() error {
config := common.NewConfig()
err := config.LoadConfig(c.ConfigFile)
if err != nil {
return err
}
c.Config = config
return nil
}
//nolint:lll
type RegisterCommand struct {
context *cli.Context
network common.Network
reader *bufio.Reader
registered bool
configOptions
ConfigTemplate configTemplate `namespace:"template"`
TagList string `long:"tag-list" env:"RUNNER_TAG_LIST" description:"Tag list"`
NonInteractive bool `short:"n" long:"non-interactive" env:"REGISTER_NON_INTERACTIVE" description:"Run registration unattended"`
LeaveRunner bool `long:"leave-runner" env:"REGISTER_LEAVE_RUNNER" description:"Don't remove runner if registration fails"`
RegistrationToken string `short:"r" long:"registration-token" env:"REGISTRATION_TOKEN" description:"Runner's registration token"`
RunUntagged bool `long:"run-untagged" env:"REGISTER_RUN_UNTAGGED" description:"Register to run untagged builds; defaults to 'true' when 'tag-list' is empty"`
Locked bool `long:"locked" env:"REGISTER_LOCKED" description:"Lock Runner for current project, defaults to 'true'"`
AccessLevel string `long:"access-level" env:"REGISTER_ACCESS_LEVEL" description:"Set access_level of the runner to not_protected or ref_protected; defaults to not_protected"`
MaximumTimeout int `long:"maximum-timeout" env:"REGISTER_MAXIMUM_TIMEOUT" description:"What is the maximum timeout (in seconds) that will be set for job when using this Runner"`
Paused bool `long:"paused" env:"REGISTER_PAUSED" description:"Set Runner to be paused, defaults to 'false'"`
common.RunnerConfig
}
type AccessLevel string
const (
NotProtected AccessLevel = "not_protected"
RefProtected AccessLevel = "ref_protected"
)
const (
defaultDockerWindowCacheDir = "c:\\cache"
)
func (s *RegisterCommand) askOnce(prompt string, result *string, allowEmpty bool) bool {
println(prompt)
if *result != "" {
print("["+*result, "]: ")
}
if s.reader == nil {
s.reader = bufio.NewReader(os.Stdin)
}
data, _, err := s.reader.ReadLine()
if err != nil {
panic(err)
}
newResult := string(data)
newResult = strings.TrimSpace(newResult)
if newResult != "" {
*result = newResult
return true
}
if allowEmpty || *result != "" {
return true
}
return false
}
func (s *RegisterCommand) ask(key, prompt string, allowEmptyOptional ...bool) string {
allowEmpty := len(allowEmptyOptional) > 0 && allowEmptyOptional[0]
result := s.context.String(key)
result = strings.TrimSpace(result)
if s.NonInteractive || prompt == "" {
if result == "" && !allowEmpty {
logrus.Panicln("The", key, "needs to be entered")
}
return result
}
for {
if s.askOnce(prompt, &result, allowEmpty) {
break
}
}
return result
}
func (s *RegisterCommand) askExecutor() {
for {
names := common.GetExecutorNames()
executors := strings.Join(names, ", ")
s.Executor = s.ask("executor", "Enter an executor: "+executors+":", true)
if common.GetExecutorProvider(s.Executor) != nil {
return
}
message := "Invalid executor specified"
if s.NonInteractive {
logrus.Panicln(message)
} else {
logrus.Errorln(message)
}
}
}
func (s *RegisterCommand) askDocker() {
s.askBasicDocker("ruby:2.6")
for _, volume := range s.Docker.Volumes {
parts := strings.Split(volume, ":")
if parts[len(parts)-1] == "/cache" {
return
}
}
if !s.Docker.DisableCache {
s.Docker.Volumes = append(s.Docker.Volumes, "/cache")
}
}
func (s *RegisterCommand) askDockerWindows() {
s.askBasicDocker("mcr.microsoft.com/windows/servercore:1809")
for _, volume := range s.Docker.Volumes {
// This does not cover all the possibilities since we don't have access
// to volume parsing package since it's internal.
if strings.Contains(volume, defaultDockerWindowCacheDir) {
return
}
}
s.Docker.Volumes = append(s.Docker.Volumes, defaultDockerWindowCacheDir)
}
func (s *RegisterCommand) askBasicDocker(exampleHelperImage string) {
if s.Docker == nil {
s.Docker = &common.DockerConfig{}
}
s.Docker.Image = s.ask(
"docker-image",
fmt.Sprintf("Enter the default Docker image (for example, %s):", exampleHelperImage),
)
}
func (s *RegisterCommand) askParallels() {
s.Parallels.BaseName = s.ask("parallels-base-name", "Enter the Parallels VM (for example, my-vm):")
}
func (s *RegisterCommand) askVirtualBox() {
s.VirtualBox.BaseName = s.ask("virtualbox-base-name", "Enter the VirtualBox VM (for example, my-vm):")
}
func (s *RegisterCommand) askSSHServer() {
s.SSH.Host = s.ask("ssh-host", "Enter the SSH server address (for example, my.server.com):")
s.SSH.Port = s.ask("ssh-port", "Enter the SSH server port (for example, 22):", true)
}
func (s *RegisterCommand) askSSHLogin() {
s.SSH.User = s.ask("ssh-user", "Enter the SSH user (for example, root):")
s.SSH.Password = s.ask(
"ssh-password",
"Enter the SSH password (for example, docker.io):",
true,
)
s.SSH.IdentityFile = s.ask(
"ssh-identity-file",
"Enter the path to the SSH identity file (for example, /home/user/.ssh/id_rsa):",
true,
)
}
func (s *RegisterCommand) addRunner(runner *common.RunnerConfig) {
s.config.Runners = append(s.config.Runners, runner)
}
func (s *RegisterCommand) askRunner() {
s.URL = s.ask("url", "Enter the GitLab instance URL (for example, https://gitlab.com/):")
if s.Token != "" {
logrus.Infoln("Token specified trying to verify runner...")
logrus.Warningln("If you want to register use the '-r' instead of '-t'.")
if !s.network.VerifyRunner(s.RunnerCredentials) {
logrus.Panicln("Failed to verify the runner. You may be having network problems.")
}
return
}
// we store registration token as token, since we pass that to RunnerCredentials
s.Token = s.ask("registration-token", "Enter the registration token:")
s.Name = s.ask("name", "Enter a description for the runner:")
s.TagList = s.ask("tag-list", "Enter tags for the runner (comma-separated):", true)
if s.TagList == "" {
s.RunUntagged = true
}
parameters := common.RegisterRunnerParameters{
Description: s.Name,
Tags: s.TagList,
Locked: s.Locked,
AccessLevel: s.AccessLevel,
RunUntagged: s.RunUntagged,
MaximumTimeout: s.MaximumTimeout,
Active: !s.Paused,
}
result := s.network.RegisterRunner(s.RunnerCredentials, parameters)
if result == nil {
logrus.Panicln("Failed to register the runner. You may be having network problems.")
}
s.Token = result.Token
s.registered = true
}
//nolint:funlen
func (s *RegisterCommand) askExecutorOptions() {
kubernetes := s.Kubernetes
machine := s.Machine
docker := s.Docker
ssh := s.SSH
parallels := s.Parallels
virtualbox := s.VirtualBox
custom := s.Custom
s.Kubernetes = nil
s.Machine = nil
s.Docker = nil
s.SSH = nil
s.Parallels = nil
s.VirtualBox = nil
s.Custom = nil
s.Referees = nil
executorFns := map[string]func(){
"kubernetes": func() {
s.Kubernetes = kubernetes
},
"docker+machine": func() {
s.Machine = machine
s.Docker = docker
s.askDocker()
},
"docker-ssh+machine": func() {
s.Machine = machine
s.Docker = docker
s.SSH = ssh
s.askDocker()
s.askSSHLogin()
},
"docker": func() {
s.Docker = docker
s.askDocker()
},
"docker-windows": func() {
if s.RunnerConfig.Shell == "" {
s.Shell = shells.SNPwsh
}
s.Docker = docker
s.askDockerWindows()
},
"docker-ssh": func() {
s.Docker = docker
s.SSH = ssh
s.askDocker()
s.askSSHLogin()
},
"ssh": func() {
s.SSH = ssh
s.askSSHServer()
s.askSSHLogin()
},
"parallels": func() {
s.SSH = ssh
s.Parallels = parallels
s.askParallels()
s.askSSHServer()
},
"virtualbox": func() {
s.SSH = ssh
s.VirtualBox = virtualbox
s.askVirtualBox()
s.askSSHLogin()
},
"shell": func() {
if runtime.GOOS == osTypeWindows && s.RunnerConfig.Shell == "" {
s.Shell = shells.SNPwsh
}
},
"custom": func() {
s.Custom = custom
},
}
executorFn, ok := executorFns[s.Executor]
if ok {
executorFn()
}
}
func (s *RegisterCommand) Execute(context *cli.Context) {
userModeWarning(true)
s.context = context
err := s.loadConfig()
if err != nil {
logrus.Panicln(err)
}
validAccessLevels := []AccessLevel{NotProtected, RefProtected}
if !accessLevelValid(validAccessLevels, AccessLevel(s.AccessLevel)) {
logrus.Panicln("Given access-level is not valid. " +
"Refer to gitlab-runner register -h for the correct options.")
}
s.askRunner()
if !s.LeaveRunner {
defer s.unregisterRunner()()
}
if s.config.Concurrent < s.Limit {
logrus.Warningf(
"Specified limit (%d) larger then current concurrent limit (%d). "+
"Concurrent limit will not be enlarged.",
s.Limit,
s.config.Concurrent,
)
}
s.askExecutor()
s.askExecutorOptions()
s.mergeTemplate()
s.addRunner(&s.RunnerConfig)
err = s.saveConfig()
if err != nil {
logrus.Panicln(err)
}
logrus.Printf(
"Runner registered successfully. " +
"Feel free to start it, but if it's running already the config should be automatically reloaded!")
}
func (s *RegisterCommand) unregisterRunner() func() {
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt)
go func() {
signal := <-signals
s.network.UnregisterRunner(s.RunnerCredentials)
logrus.Fatalf("RECEIVED SIGNAL: %v", signal)
}()
return func() {
// De-register runner on panic
if r := recover(); r != nil {
if s.registered {
s.network.UnregisterRunner(s.RunnerCredentials)
}
// pass panic to next defer
panic(r)
}
}
}
func (s *RegisterCommand) mergeTemplate() {
if !s.ConfigTemplate.Enabled() {
return
}
logrus.Infof("Merging configuration from template file %q", s.ConfigTemplate.ConfigFile)
err := s.ConfigTemplate.MergeTo(&s.RunnerConfig)
if err != nil {
logrus.WithError(err).Fatal("Could not handle configuration merging from template file")
}
}
func getHostname() string {
hostname, _ := os.Hostname()
return hostname
}
func newRegisterCommand() *RegisterCommand {
return &RegisterCommand{
RunnerConfig: common.RunnerConfig{
Name: getHostname(),
RunnerSettings: common.RunnerSettings{
Kubernetes: &common.KubernetesConfig{},
Cache: &common.CacheConfig{},
Machine: &common.DockerMachine{},
Docker: &common.DockerConfig{},
SSH: &ssh.Config{},
Parallels: &common.ParallelsConfig{},
VirtualBox: &common.VirtualBoxConfig{},
},
},
Locked: true,
Paused: false,
network: network.NewGitLabClient(),
}
}
func accessLevelValid(levels []AccessLevel, givenLevel AccessLevel) bool {
if givenLevel == "" {
return true
}
for _, level := range levels {
if givenLevel == level {
return true
}
}
return false
}
func init() {
common.RegisterCommand2("register", "register a new runner", newRegisterCommand())
}
package commands
import (
"fmt"
"os"
"runtime"
"github.com/kardianos/service"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
service_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/service"
)
const (
defaultServiceName = "gitlab-runner"
defaultDescription = "GitLab Runner"
)
type NullService struct {
}
func (n *NullService) Start(s service.Service) error {
return nil
}
func (n *NullService) Stop(s service.Service) error {
return nil
}
func runServiceInstall(s service.Service, c *cli.Context) error {
if user := c.String("user"); user == "" && os.Getuid() == 0 {
logrus.Fatal("Please specify user that will run gitlab-runner service")
}
if configFile := c.String("config"); configFile != "" {
// try to load existing config
config := common.NewConfig()
err := config.LoadConfig(configFile)
if err != nil {
return err
}
// save config for the first time
if !config.Loaded {
err = config.SaveConfig(configFile)
if err != nil {
return err
}
}
}
return service.Control(s, "install")
}
func runServiceStatus(displayName string, s service.Service) {
status, err := s.Status()
description := ""
switch status {
case service.StatusRunning:
description = "Service is running"
case service.StatusStopped:
description = "Service has stopped"
default:
description = "Service status unknown"
if err != nil {
description = err.Error()
}
}
if status != service.StatusRunning {
fmt.Fprintf(os.Stderr, "%s: %s\n", displayName, description)
os.Exit(1)
}
fmt.Printf("%s: %s\n", displayName, description)
}
func GetServiceArguments(c *cli.Context) (arguments []string) {
if wd := c.String("working-directory"); wd != "" {
arguments = append(arguments, "--working-directory", wd)
}
if config := c.String("config"); config != "" {
arguments = append(arguments, "--config", config)
}
if sn := c.String("service"); sn != "" {
arguments = append(arguments, "--service", sn)
}
// syslogging doesn't make sense for systemd systems as those log straight to journald
syslog := !c.IsSet("syslog") || c.Bool("syslog")
if service.Platform() == "linux-systemd" && !c.IsSet("syslog") {
syslog = false
}
if syslog {
arguments = append(arguments, "--syslog")
}
return
}
func createServiceConfig(c *cli.Context) *service.Config {
config := &service.Config{
Name: c.String("service"),
DisplayName: c.String("service"),
Description: defaultDescription,
Arguments: append([]string{"run"}, GetServiceArguments(c)...),
}
// setup os specific service config
setupOSServiceConfig(c, config)
return config
}
func RunServiceControl(c *cli.Context) {
svcConfig := createServiceConfig(c)
s, err := service_helpers.New(&NullService{}, svcConfig)
if err != nil {
logrus.Fatal(err)
}
switch c.Command.Name {
case "install":
err = runServiceInstall(s, c)
case "status":
runServiceStatus(svcConfig.DisplayName, s)
default:
err = service.Control(s, c.Command.Name)
}
if err != nil {
logrus.Fatal(err)
}
}
func GetFlags() []cli.Flag {
return []cli.Flag{
cli.StringFlag{
Name: "service, n",
Value: defaultServiceName,
Usage: "Specify service name to use",
},
}
}
func GetInstallFlags() []cli.Flag {
installFlags := GetFlags()
installFlags = append(
installFlags,
cli.StringFlag{
Name: "working-directory, d",
Value: helpers.GetCurrentWorkingDirectory(),
Usage: "Specify custom root directory where all data are stored",
},
cli.StringFlag{
Name: "config, c",
Value: GetDefaultConfigFile(),
Usage: "Specify custom config file",
},
cli.BoolFlag{
Name: "syslog",
Usage: "Setup system logging integration",
},
)
if runtime.GOOS == osTypeWindows {
installFlags = append(
installFlags,
cli.StringFlag{
Name: "user, u",
Value: "",
Usage: "Specify user-name to secure the runner",
},
cli.StringFlag{
Name: "password, p",
Value: "",
Usage: "Specify user password to install service (required)",
})
} else if os.Getuid() == 0 {
installFlags = append(installFlags, cli.StringFlag{
Name: "user, u",
Value: "",
Usage: "Specify user-name to secure the runner",
})
}
return installFlags
}
func init() {
flags := GetFlags()
installFlags := GetInstallFlags()
common.RegisterCommand(cli.Command{
Name: "install",
Usage: "install service",
Action: RunServiceControl,
Flags: installFlags,
})
common.RegisterCommand(cli.Command{
Name: "uninstall",
Usage: "uninstall service",
Action: RunServiceControl,
Flags: flags,
})
common.RegisterCommand(cli.Command{
Name: "start",
Usage: "start service",
Action: RunServiceControl,
Flags: flags,
})
common.RegisterCommand(cli.Command{
Name: "stop",
Usage: "stop service",
Action: RunServiceControl,
Flags: flags,
})
common.RegisterCommand(cli.Command{
Name: "restart",
Usage: "restart service",
Action: RunServiceControl,
Flags: flags,
})
common.RegisterCommand(cli.Command{
Name: "status",
Usage: "get status of a service",
Action: RunServiceControl,
Flags: flags,
})
}
package commands
import (
"os"
"github.com/kardianos/service"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
service_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/service"
)
func setupOSServiceConfig(c *cli.Context, config *service.Config) {
if os.Getuid() != 0 {
logrus.Fatal("The --user is not supported for non-root users")
}
user := c.String("user")
if user != "" {
config.Arguments = append(config.Arguments, "--user", user)
}
switch service.Platform() {
case "linux-systemd":
config.Dependencies = []string{
"After=syslog.target network.target",
}
config.Option = service.KeyValue{
"Restart": "always",
}
case "unix-systemv":
script := service_helpers.SysvScript()
if script != "" {
config.Option = service.KeyValue{
"SysvScript": script,
}
}
}
}
package commands
import (
"context"
"os"
"os/signal"
"syscall"
"time"
"github.com/sirupsen/logrus"
"github.com/tevino/abool"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/network"
)
type RunSingleCommand struct {
common.RunnerConfig
network common.Network
WaitTimeout int `long:"wait-timeout" description:"How long to wait in seconds before receiving the first job"`
lastBuild time.Time
runForever bool
MaxBuilds int `long:"max-builds" description:"How many builds to process before exiting"`
finished *abool.AtomicBool
interruptSignals chan os.Signal
}
func waitForInterrupts(
finished *abool.AtomicBool,
abortSignal chan os.Signal,
doneSignal chan int,
interruptSignals chan os.Signal,
) {
if interruptSignals == nil {
interruptSignals = make(chan os.Signal)
}
signal.Notify(interruptSignals, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
interrupt := <-interruptSignals
if finished != nil {
finished.Set()
}
// request stop, but wait for force exit
for interrupt == syscall.SIGQUIT {
logrus.Warningln("Requested quit, waiting for builds to finish")
interrupt = <-interruptSignals
}
logrus.Warningln("Requested exit:", interrupt)
go func() {
for {
abortSignal <- interrupt
}
}()
select {
case newSignal := <-interruptSignals:
logrus.Fatalln("forced exit:", newSignal)
case <-time.After(common.ShutdownTimeout * time.Second):
logrus.Fatalln("shutdown timed out")
case <-doneSignal:
}
}
// Things to do after a build
func (r *RunSingleCommand) postBuild() {
if r.MaxBuilds > 0 {
r.MaxBuilds--
}
r.lastBuild = time.Now()
}
func (r *RunSingleCommand) processBuild(data common.ExecutorData, abortSignal chan os.Signal) error {
jobData, healthy := r.network.RequestJob(context.Background(), r.RunnerConfig, nil)
if !healthy {
logrus.Println("Runner is not healthy!")
select {
case <-time.After(common.NotHealthyCheckInterval * time.Second):
case <-abortSignal:
}
return nil
}
if jobData == nil {
select {
case <-time.After(common.CheckInterval):
case <-abortSignal:
}
return nil
}
config := common.NewConfig()
newBuild, err := common.NewBuild(*jobData, &r.RunnerConfig, abortSignal, data)
if err != nil {
return err
}
jobCredentials := &common.JobCredentials{
ID: jobData.ID,
Token: jobData.Token,
}
trace, err := r.network.ProcessJob(r.RunnerConfig, jobCredentials)
if err != nil {
return err
}
defer trace.Success()
err = newBuild.Run(config, trace)
r.postBuild()
return err
}
func (r *RunSingleCommand) checkFinishedConditions() {
if r.MaxBuilds < 1 && !r.runForever {
logrus.Println("This runner has processed its build limit, so now exiting")
r.finished.Set()
}
if r.WaitTimeout > 0 && int(time.Since(r.lastBuild).Seconds()) > r.WaitTimeout {
logrus.Println("This runner has not received a job in", r.WaitTimeout, "seconds, so now exiting")
r.finished.Set()
}
}
func (r *RunSingleCommand) Execute(c *cli.Context) {
if r.URL == "" {
logrus.Fatalln("Missing URL")
}
if r.Token == "" {
logrus.Fatalln("Missing Token")
}
if r.Executor == "" {
logrus.Fatalln("Missing Executor")
}
executorProvider := common.GetExecutorProvider(r.Executor)
if executorProvider == nil {
logrus.Fatalln("Unknown executor:", r.Executor)
}
logrus.Println("Starting runner for", r.URL, "with token", r.ShortDescription(), "...")
r.finished = abool.New()
abortSignal := make(chan os.Signal)
doneSignal := make(chan int, 1)
r.runForever = r.MaxBuilds == 0
go waitForInterrupts(r.finished, abortSignal, doneSignal, r.interruptSignals)
r.lastBuild = time.Now()
for !r.finished.IsSet() {
data, err := executorProvider.Acquire(&r.RunnerConfig)
if err != nil {
logrus.Warningln("Executor update:", err)
}
pErr := r.processBuild(data, abortSignal)
if pErr != nil {
logrus.WithError(pErr).Error("Failed to process build")
}
r.checkFinishedConditions()
executorProvider.Release(&r.RunnerConfig, data)
}
doneSignal <- 0
}
func init() {
common.RegisterCommand2("run-single", "start single runner", &RunSingleCommand{
network: network.NewGitLabClient(),
})
}
package commands
import (
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/network"
)
//nolint:lll
type UnregisterCommand struct {
configOptions
common.RunnerCredentials
network common.Network
Name string `toml:"name" json:"name" short:"n" long:"name" description:"Name of the runner you wish to unregister"`
AllRunners bool `toml:"all_runners" json:"all-runners" long:"all-runners" description:"Unregister all runners"`
}
func (c *UnregisterCommand) unregisterAllRunners() (runners []*common.RunnerConfig) {
logrus.Warningln("Unregistering all runners")
for _, r := range c.config.Runners {
if !c.network.UnregisterRunner(r.RunnerCredentials) {
logrus.Errorln("Failed to unregister runner", r.Name)
// If unregister fails, leave the runner in the config
runners = append(runners, r)
}
}
return
}
func (c *UnregisterCommand) unregisterSingleRunner() []*common.RunnerConfig {
if len(c.Name) > 0 { // Unregister when given a name
runnerConfig, err := c.RunnerByName(c.Name)
if err != nil {
logrus.Fatalln(err)
}
c.RunnerCredentials = runnerConfig.RunnerCredentials
}
// Unregister given Token and URL of the runner
if !c.network.UnregisterRunner(c.RunnerCredentials) {
logrus.Fatalln("Failed to unregister runner", c.Name)
}
var runners []*common.RunnerConfig
for _, otherRunner := range c.config.Runners {
if otherRunner.RunnerCredentials != c.RunnerCredentials {
runners = append(runners, otherRunner)
}
}
return runners
}
func (c *UnregisterCommand) Execute(context *cli.Context) {
userModeWarning(false)
err := c.loadConfig()
if err != nil {
logrus.Fatalln(err)
return
}
var runners []*common.RunnerConfig
if c.AllRunners {
runners = c.unregisterAllRunners()
} else {
runners = c.unregisterSingleRunner()
}
// check if anything changed
if len(c.config.Runners) == len(runners) {
return
}
c.config.Runners = runners
// save config file
err = c.saveConfig()
if err != nil {
logrus.Fatalln("Failed to update", c.ConfigFile, err)
}
logrus.Println("Updated", c.ConfigFile)
}
func init() {
common.RegisterCommand2("unregister", "unregister specific runner", &UnregisterCommand{
network: network.NewGitLabClient(),
})
}
package commands
import (
"os"
"runtime"
"github.com/sirupsen/logrus"
)
func userModeWarning(withRun bool) {
logrus.WithFields(logrus.Fields{
"GOOS": runtime.GOOS,
"uid": os.Getuid(),
}).Debugln("Checking runtime mode")
// everything is supported on windows
if runtime.GOOS == osTypeWindows {
return
}
systemMode := os.Getuid() == 0
// We support services on Linux, Windows and Darwin
noServices :=
runtime.GOOS != osTypeLinux &&
runtime.GOOS != osTypeDarwin
// We don't support services installed as an User on Linux
noUserService :=
!systemMode &&
runtime.GOOS == osTypeLinux
if systemMode {
logrus.Infoln("Running in system-mode.")
} else {
logrus.Warningln("Running in user-mode.")
}
if withRun {
if noServices {
logrus.Warningln("You need to manually start builds processing:")
logrus.Warningln("$ gitlab-runner run")
} else if noUserService {
logrus.Warningln("The user-mode requires you to manually start builds processing:")
logrus.Warningln("$ gitlab-runner run")
}
}
if !systemMode {
logrus.Warningln("Use sudo for system-mode:")
logrus.Warningln("$ sudo gitlab-runner...")
}
logrus.Infoln("")
}
package commands
import (
"errors"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/network"
)
//nolint:lll
type VerifyCommand struct {
configOptions
common.RunnerCredentials
network common.Network
Name string `toml:"name" json:"name" short:"n" long:"name" description:"Name of the runner you wish to verify"`
DeleteNonExisting bool `long:"delete" description:"Delete no longer existing runners?"`
}
func (c *VerifyCommand) Execute(context *cli.Context) {
userModeWarning(true)
err := c.loadConfig()
if err != nil {
logrus.Fatalln(err)
return
}
// check if there's something to verify
toVerify, okRunners, err := c.selectRunners()
if err != nil {
logrus.Fatalln(err)
return
}
// verify if runner exist
for _, runner := range toVerify {
if c.network.VerifyRunner(runner.RunnerCredentials) {
okRunners = append(okRunners, runner)
}
}
// check if anything changed
if len(c.config.Runners) == len(okRunners) {
return
}
if !c.DeleteNonExisting {
logrus.Fatalln("Failed to verify runners")
return
}
c.config.Runners = okRunners
// save config file
err = c.saveConfig()
if err != nil {
logrus.Fatalln("Failed to update", c.ConfigFile, err)
}
logrus.Println("Updated", c.ConfigFile)
}
func (c *VerifyCommand) selectRunners() (toVerify, okRunners []*common.RunnerConfig, err error) {
var selectorPresent = c.Name != "" || c.RunnerCredentials.URL != "" || c.RunnerCredentials.Token != ""
for _, runner := range c.config.Runners {
selected := !selectorPresent || runner.Name == c.Name || runner.RunnerCredentials.SameAs(&c.RunnerCredentials)
if selected {
toVerify = append(toVerify, runner)
} else {
okRunners = append(okRunners, runner)
}
}
if selectorPresent && len(toVerify) == 0 {
err = errors.New("no runner matches the filtering parameters")
}
return
}
func init() {
common.RegisterCommand2("verify", "verify all registered runners", &VerifyCommand{
network: network.NewGitLabClient(),
})
}
package common
import (
"errors"
"fmt"
"github.com/bmatcuk/doublestar"
)
type VerifyAllowedImageOptions struct {
Image string
OptionName string
AllowedImages []string
InternalImages []string
}
var ErrDisallowedImage = errors.New("disallowed image")
func VerifyAllowedImage(options VerifyAllowedImageOptions, logger BuildLogger) error {
for _, allowedImage := range options.AllowedImages {
ok, _ := doublestar.Match(allowedImage, options.Image)
if ok {
return nil
}
}
for _, internalImage := range options.InternalImages {
if internalImage == options.Image {
return nil
}
}
if len(options.AllowedImages) != 0 {
logger.Println()
logger.Errorln(
fmt.Sprintf("The %q image is not present on list of allowed %s:", options.Image, options.OptionName),
)
for _, allowedImage := range options.AllowedImages {
logger.Println("-", allowedImage)
}
logger.Println()
} else {
// by default allow to override the image name
return nil
}
errorMsg := `Please check runner's configuration:
https://docs.gitlab.com/runner/configuration/advanced-configuration.html
#restricting-docker-images-and-services`
logger.Println(errorMsg)
return ErrDisallowedImage
}
package common
import (
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"net/url"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
"gitlab.com/gitlab-org/gitlab-runner/helpers/dns"
"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
"gitlab.com/gitlab-org/gitlab-runner/helpers/tls"
"gitlab.com/gitlab-org/gitlab-runner/referees"
"gitlab.com/gitlab-org/gitlab-runner/session"
"gitlab.com/gitlab-org/gitlab-runner/session/proxy"
"gitlab.com/gitlab-org/gitlab-runner/session/terminal"
)
type GitStrategy int
const (
GitClone GitStrategy = iota
GitFetch
GitNone
)
const (
gitCleanFlagsDefault = "-ffdx"
gitCleanFlagsNone = "none"
)
const (
gitFetchFlagsDefault = "--prune --quiet"
gitFetchFlagsNone = "none"
)
type SubmoduleStrategy int
const (
SubmoduleInvalid SubmoduleStrategy = iota
SubmoduleNone
SubmoduleNormal
SubmoduleRecursive
)
type BuildRuntimeState string
const (
BuildRunStatePending BuildRuntimeState = "pending"
BuildRunRuntimeRunning BuildRuntimeState = "running"
BuildRunRuntimeSuccess BuildRuntimeState = "success"
BuildRunRuntimeFailed BuildRuntimeState = "failed"
BuildRunRuntimeCanceled BuildRuntimeState = "canceled"
BuildRunRuntimeTerminated BuildRuntimeState = "terminated"
BuildRunRuntimeTimedout BuildRuntimeState = "timedout"
)
type BuildStage string
const (
BuildStageResolveSecrets BuildStage = "resolve_secrets"
BuildStagePrepareExecutor BuildStage = "prepare_executor"
BuildStagePrepare BuildStage = "prepare_script"
BuildStageGetSources BuildStage = "get_sources"
BuildStageRestoreCache BuildStage = "restore_cache"
BuildStageDownloadArtifacts BuildStage = "download_artifacts"
BuildStageAfterScript BuildStage = "after_script"
BuildStageArchiveOnSuccessCache BuildStage = "archive_cache"
BuildStageArchiveOnFailureCache BuildStage = "archive_cache_on_failure"
BuildStageUploadOnSuccessArtifacts BuildStage = "upload_artifacts_on_success"
BuildStageUploadOnFailureArtifacts BuildStage = "upload_artifacts_on_failure"
BuildStageCleanupFileVariables BuildStage = "cleanup_file_variables"
)
// staticBuildStages is a list of BuildStages which are executed on every build
// and are not dynamically generated from steps.
var staticBuildStages = []BuildStage{
BuildStagePrepare,
BuildStageGetSources,
BuildStageRestoreCache,
BuildStageDownloadArtifacts,
BuildStageAfterScript,
BuildStageArchiveOnSuccessCache,
BuildStageArchiveOnFailureCache,
BuildStageUploadOnSuccessArtifacts,
BuildStageUploadOnFailureArtifacts,
BuildStageCleanupFileVariables,
}
const (
ExecutorJobSectionAttempts = "EXECUTOR_JOB_SECTION_ATTEMPTS"
)
// ErrSkipBuildStage is returned when there's nothing to be executed for the
// build stage.
var ErrSkipBuildStage = errors.New("skip build stage")
type invalidAttemptError struct {
key string
}
func (i *invalidAttemptError) Error() string {
return fmt.Sprintf("number of attempts out of the range [1, 10] for variable: %s", i.key)
}
func (i *invalidAttemptError) Is(err error) bool {
_, ok := err.(*invalidAttemptError)
return ok
}
type Build struct {
JobResponse `yaml:",inline"`
SystemInterrupt chan os.Signal `json:"-" yaml:"-"`
RootDir string `json:"-" yaml:"-"`
BuildDir string `json:"-" yaml:"-"`
CacheDir string `json:"-" yaml:"-"`
Hostname string `json:"-" yaml:"-"`
Runner *RunnerConfig `json:"runner"`
ExecutorData ExecutorData
ExecutorFeatures FeaturesInfo `json:"-" yaml:"-"`
// Unique ID for all running builds on this runner
RunnerID int `json:"runner_id"`
// Unique ID for all running builds on this runner and this project
ProjectRunnerID int `json:"project_runner_id"`
// statusLock handles access to currentStage, currentState and
// executorStageResolver. These variables can be accessed via
// CurrentStage(), CurrentState() and CurrentExecutorStage() from the
// metrics go routine whilst a build is in-flight.
statusLock sync.RWMutex
currentStage BuildStage
currentState BuildRuntimeState
executorStageResolver func() ExecutorStage
secretsResolver func(l logger, registry SecretResolverRegistry) (SecretsResolver, error)
Session *session.Session
logger BuildLogger
allVariables JobVariables
secretsVariables JobVariables
createdAt time.Time
Referees []referees.Referee
ArtifactUploader func(config JobCredentials, reader io.ReadCloser, options ArtifactsOptions) UploadState
}
func (b *Build) setCurrentStage(stage BuildStage) {
b.statusLock.Lock()
defer b.statusLock.Unlock()
b.currentStage = stage
}
func (b *Build) CurrentStage() BuildStage {
b.statusLock.RLock()
defer b.statusLock.RUnlock()
return b.currentStage
}
func (b *Build) setCurrentState(state BuildRuntimeState) {
b.statusLock.Lock()
defer b.statusLock.Unlock()
b.currentState = state
}
func (b *Build) CurrentState() BuildRuntimeState {
b.statusLock.RLock()
defer b.statusLock.RUnlock()
return b.currentState
}
func (b *Build) Log() *logrus.Entry {
return b.Runner.Log().WithField("job", b.ID).WithField("project", b.JobInfo.ProjectID)
}
func (b *Build) ProjectUniqueName() string {
projectUniqueName := fmt.Sprintf(
"runner-%s-project-%d-concurrent-%d",
b.Runner.ShortDescription(),
b.JobInfo.ProjectID,
b.ProjectRunnerID,
)
return dns.MakeRFC1123Compatible(projectUniqueName)
}
func (b *Build) ProjectSlug() (string, error) {
url, err := url.Parse(b.GitInfo.RepoURL)
if err != nil {
return "", err
}
if url.Host == "" {
return "", errors.New("only URI reference supported")
}
slug := url.Path
slug = strings.TrimSuffix(slug, ".git")
slug = path.Clean(slug)
if slug == "." {
return "", errors.New("invalid path")
}
if strings.Contains(slug, "..") {
return "", errors.New("it doesn't look like a valid path")
}
return slug, nil
}
func (b *Build) ProjectUniqueDir(sharedDir bool) string {
dir, err := b.ProjectSlug()
if err != nil {
dir = fmt.Sprintf("project-%d", b.JobInfo.ProjectID)
}
// for shared dirs path is constructed like this:
// <some-path>/runner-short-id/concurrent-id/group-name/project-name/
// ex.<some-path>/01234567/0/group/repo/
if sharedDir {
dir = path.Join(
b.Runner.ShortDescription(),
fmt.Sprintf("%d", b.ProjectRunnerID),
dir,
)
}
return dir
}
func (b *Build) FullProjectDir() string {
return helpers.ToSlash(b.BuildDir)
}
func (b *Build) TmpProjectDir() string {
return helpers.ToSlash(b.BuildDir) + ".tmp"
}
// BuildStages returns a list of all BuildStages which will be executed.
// Not in the order of execution.
func (b *Build) BuildStages() []BuildStage {
stages := make([]BuildStage, len(staticBuildStages))
copy(stages, staticBuildStages)
for _, s := range b.Steps {
if s.Name == StepNameAfterScript {
continue
}
stages = append(stages, StepToBuildStage(s))
}
return stages
}
func (b *Build) getCustomBuildDir(rootDir, overrideKey string, customBuildDirEnabled, sharedDir bool) (string, error) {
dir := b.GetAllVariables().Get(overrideKey)
if dir == "" {
return path.Join(rootDir, b.ProjectUniqueDir(sharedDir)), nil
}
if !customBuildDirEnabled {
return "", MakeBuildError("setting %s is not allowed, enable `custom_build_dir` feature", overrideKey)
}
relDir, err := filepath.Rel(rootDir, dir)
if err != nil {
return "", &BuildError{Inner: err}
}
if strings.HasPrefix(relDir, "..") {
return "", MakeBuildError("the %s=%q has to be within %q", overrideKey, dir, rootDir)
}
return dir, nil
}
func (b *Build) StartBuild(rootDir, cacheDir string, customBuildDirEnabled, sharedDir bool) error {
if rootDir == "" {
return MakeBuildError("the builds_dir is not configured")
}
if cacheDir == "" {
return MakeBuildError("the cache_dir is not configured")
}
// We set RootDir and invalidate variables
// to be able to use CI_BUILDS_DIR
b.RootDir = rootDir
b.CacheDir = path.Join(cacheDir, b.ProjectUniqueDir(false))
b.refreshAllVariables()
var err error
b.BuildDir, err = b.getCustomBuildDir(b.RootDir, "GIT_CLONE_PATH", customBuildDirEnabled, sharedDir)
if err != nil {
return err
}
// We invalidate variables to be able to use
// CI_CACHE_DIR and CI_PROJECT_DIR
b.refreshAllVariables()
return nil
}
func (b *Build) executeStage(ctx context.Context, buildStage BuildStage, executor Executor) error {
if ctx.Err() != nil {
return ctx.Err()
}
b.setCurrentStage(buildStage)
b.Log().WithField("build_stage", buildStage).Debug("Executing build stage")
shell := executor.Shell()
if shell == nil {
return errors.New("no shell defined")
}
script, err := GenerateShellScript(buildStage, *shell)
if errors.Is(err, ErrSkipBuildStage) {
if b.IsFeatureFlagOn(featureflags.SkipNoOpBuildStages) {
b.Log().WithField("build_stage", buildStage).Debug("Skipping stage (nothing to do)")
return nil
}
err = nil
}
if err != nil {
return err
}
// Nothing to execute
if script == "" {
return nil
}
cmd := ExecutorCommand{
Context: ctx,
Script: script,
Stage: buildStage,
Predefined: getPredefinedEnv(buildStage),
}
section := helpers.BuildSection{
Name: string(buildStage),
SkipMetrics: !b.JobResponse.Features.TraceSections,
Run: func() error {
msg := fmt.Sprintf(
"%s%s%s",
helpers.ANSI_BOLD_CYAN,
GetStageDescription(buildStage),
helpers.ANSI_RESET,
)
b.logger.Println(msg)
return executor.Run(cmd)
},
}
return section.Execute(&b.logger)
}
// getPredefinedEnv returns whether a stage should be executed on
// the predefined environment that GitLab Runner provided.
func getPredefinedEnv(buildStage BuildStage) bool {
env := map[BuildStage]bool{
BuildStagePrepare: true,
BuildStageGetSources: true,
BuildStageRestoreCache: true,
BuildStageDownloadArtifacts: true,
BuildStageAfterScript: false,
BuildStageArchiveOnSuccessCache: true,
BuildStageArchiveOnFailureCache: true,
BuildStageUploadOnFailureArtifacts: true,
BuildStageUploadOnSuccessArtifacts: true,
BuildStageCleanupFileVariables: true,
}
predefined, ok := env[buildStage]
if !ok {
return false
}
return predefined
}
func GetStageDescription(stage BuildStage) string {
descriptions := map[BuildStage]string{
BuildStagePrepare: "Preparing environment",
BuildStageGetSources: "Getting source from Git repository",
BuildStageRestoreCache: "Restoring cache",
BuildStageDownloadArtifacts: "Downloading artifacts",
BuildStageAfterScript: "Running after_script",
BuildStageArchiveOnSuccessCache: "Saving cache for successful job",
BuildStageArchiveOnFailureCache: "Saving cache for failed job",
BuildStageUploadOnFailureArtifacts: "Uploading artifacts for failed job",
BuildStageUploadOnSuccessArtifacts: "Uploading artifacts for successful job",
BuildStageCleanupFileVariables: "Cleaning up file based variables",
}
description, ok := descriptions[stage]
if !ok {
return fmt.Sprintf("Executing %q stage of the job script", stage)
}
return description
}
func (b *Build) executeUploadArtifacts(ctx context.Context, state error, executor Executor) (err error) {
if state == nil {
return b.executeStage(ctx, BuildStageUploadOnSuccessArtifacts, executor)
}
return b.executeStage(ctx, BuildStageUploadOnFailureArtifacts, executor)
}
func (b *Build) executeArchiveCache(ctx context.Context, state error, executor Executor) (err error) {
if state == nil {
return b.executeStage(ctx, BuildStageArchiveOnSuccessCache, executor)
}
return b.executeStage(ctx, BuildStageArchiveOnFailureCache, executor)
}
func (b *Build) executeScript(ctx context.Context, executor Executor) error {
// track job start and create referees
startTime := time.Now()
b.createReferees(executor)
// Prepare stage
err := b.executeStage(ctx, BuildStagePrepare, executor)
if err != nil {
return fmt.Errorf(
"prepare environment: %w. "+
"Check https://docs.gitlab.com/runner/shells/index.html#shell-profile-loading for more information",
err,
)
}
err = b.attemptExecuteStage(ctx, BuildStageGetSources, executor, b.GetGetSourcesAttempts())
if err == nil {
err = b.attemptExecuteStage(ctx, BuildStageRestoreCache, executor, b.GetRestoreCacheAttempts())
}
if err == nil {
err = b.attemptExecuteStage(ctx, BuildStageDownloadArtifacts, executor, b.GetDownloadArtifactsAttempts())
}
if err == nil {
for _, s := range b.Steps {
// after_script has a separate BuildStage. See common.BuildStageAfterScript
if s.Name == StepNameAfterScript {
continue
}
err = b.executeStage(ctx, StepToBuildStage(s), executor)
if err != nil {
break
}
}
b.executeAfterScript(ctx, err, executor)
}
archiveCacheErr := b.executeArchiveCache(ctx, err, executor)
artifactUploadErr := b.executeUploadArtifacts(ctx, err, executor)
// track job end and execute referees
endTime := time.Now()
b.executeUploadReferees(ctx, startTime, endTime)
b.removeFileBasedVariables(ctx, executor)
return b.pickPriorityError(err, archiveCacheErr, artifactUploadErr)
}
func (b *Build) pickPriorityError(jobErr error, archiveCacheErr error, artifactUploadErr error) error {
// Use job's errors which came before upload errors as most important to surface
if jobErr != nil {
return jobErr
}
// Otherwise, use uploading errors
if archiveCacheErr != nil {
return archiveCacheErr
}
return artifactUploadErr
}
func (b *Build) executeAfterScript(ctx context.Context, err error, executor Executor) {
state, _ := b.runtimeStateAndError(err)
b.GetAllVariables().OverwriteKey("CI_JOB_STATUS", JobVariable{
Key: "CI_JOB_STATUS",
Value: string(state),
})
ctx, cancel := context.WithTimeout(ctx, AfterScriptTimeout)
defer cancel()
_ = b.executeStage(ctx, BuildStageAfterScript, executor)
}
// StepToBuildStage returns the BuildStage corresponding to a step.
func StepToBuildStage(s Step) BuildStage {
return BuildStage(fmt.Sprintf("step_%s", strings.ToLower(string(s.Name))))
}
func (b *Build) createReferees(executor Executor) {
b.Referees = referees.CreateReferees(executor, b.Runner.Referees, b.Log())
}
func (b *Build) removeFileBasedVariables(ctx context.Context, executor Executor) {
err := b.executeStage(ctx, BuildStageCleanupFileVariables, executor)
if err != nil {
b.Log().WithError(err).Warning("Error while executing file based variables removal script")
}
}
func (b *Build) executeUploadReferees(ctx context.Context, startTime, endTime time.Time) {
if b.Referees == nil || b.ArtifactUploader == nil {
b.Log().Debug("Skipping referees execution")
return
}
jobCredentials := JobCredentials{
ID: b.JobResponse.ID,
Token: b.JobResponse.Token,
URL: b.Runner.RunnerCredentials.URL,
}
// execute and upload the results of each referee
for _, referee := range b.Referees {
if referee == nil {
continue
}
reader, err := referee.Execute(ctx, startTime, endTime)
// keep moving even if a subset of the referees have failed
if err != nil {
continue
}
// referee ran successfully, upload its results to GitLab as an artifact
b.ArtifactUploader(jobCredentials, ioutil.NopCloser(reader), ArtifactsOptions{
BaseName: referee.ArtifactBaseName(),
Type: referee.ArtifactType(),
Format: ArtifactFormat(referee.ArtifactFormat()),
})
}
}
func (b *Build) attemptExecuteStage(
ctx context.Context,
buildStage BuildStage,
executor Executor,
attempts int,
) (err error) {
if attempts < 1 || attempts > 10 {
return fmt.Errorf("number of attempts out of the range [1, 10] for stage: %s", buildStage)
}
for attempt := 0; attempt < attempts; attempt++ {
if err = b.executeStage(ctx, buildStage, executor); err == nil {
return
}
}
return
}
func (b *Build) GetBuildTimeout() time.Duration {
buildTimeout := b.RunnerInfo.Timeout
if buildTimeout <= 0 {
buildTimeout = DefaultTimeout
}
return time.Duration(buildTimeout) * time.Second
}
func (b *Build) handleError(err error) error {
state, err := b.runtimeStateAndError(err)
b.setCurrentState(state)
return err
}
func (b *Build) runtimeStateAndError(err error) (BuildRuntimeState, error) {
switch err {
case context.Canceled:
return BuildRunRuntimeCanceled, &BuildError{
Inner: errors.New("canceled"),
FailureReason: JobCanceled,
}
case context.DeadlineExceeded:
return BuildRunRuntimeTimedout, &BuildError{
Inner: fmt.Errorf("execution took longer than %v seconds", b.GetBuildTimeout()),
FailureReason: JobExecutionTimeout,
}
case nil:
return BuildRunRuntimeSuccess, nil
default:
return BuildRunRuntimeFailed, err
}
}
func (b *Build) run(ctx context.Context, executor Executor) (err error) {
b.setCurrentState(BuildRunRuntimeRunning)
buildFinish := make(chan error, 1)
buildPanic := make(chan error, 1)
runContext, runCancel := context.WithCancel(context.Background())
defer runCancel()
if term, ok := executor.(terminal.InteractiveTerminal); b.Session != nil && ok {
b.Session.SetInteractiveTerminal(term)
}
if proxyPooler, ok := executor.(proxy.Pooler); b.Session != nil && ok {
b.Session.SetProxyPool(proxyPooler)
}
// Run build script
go func() {
defer func() {
if r := recover(); r != nil {
buildPanic <- &BuildError{FailureReason: RunnerSystemFailure, Inner: fmt.Errorf("panic: %s", r)}
}
}()
buildFinish <- b.executeScript(runContext, executor)
}()
// Wait for signals: cancel, timeout, abort or finish
b.Log().Debugln("Waiting for signals...")
select {
case <-ctx.Done():
err = b.handleError(ctx.Err())
case signal := <-b.SystemInterrupt:
err = &BuildError{
Inner: fmt.Errorf("aborted: %v", signal),
FailureReason: RunnerSystemFailure,
}
b.setCurrentState(BuildRunRuntimeTerminated)
case err = <-buildFinish:
if err != nil {
b.setCurrentState(BuildRunRuntimeFailed)
} else {
b.setCurrentState(BuildRunRuntimeSuccess)
}
return err
case err = <-buildPanic:
b.setCurrentState(BuildRunRuntimeTerminated)
return err
}
b.Log().WithError(err).Debugln("Waiting for build to finish...")
// Wait till we receive that build did finish
runCancel()
b.waitForBuildFinish(buildFinish, WaitForBuildFinishTimeout)
return err
}
// waitForBuildFinish will wait for the build to finish or timeout, whichever
// comes first. This is to prevent issues where something in the build can't be
// killed or processed and results into the Job running until the GitLab Runner
// process exists.
func (b *Build) waitForBuildFinish(buildFinish <-chan error, timeout time.Duration) {
select {
case <-buildFinish:
return
case <-time.After(timeout):
b.logger.Warningln("Timed out waiting for the build to finish")
return
}
}
func (b *Build) retryCreateExecutor(
options ExecutorPrepareOptions,
provider ExecutorProvider,
logger BuildLogger,
) (Executor, error) {
var err error
for tries := 0; tries < PreparationRetries; tries++ {
executor := provider.Create()
if executor == nil {
return nil, errors.New("failed to create executor")
}
b.setExecutorStageResolver(executor.GetCurrentStage)
err = executor.Prepare(options)
if err == nil {
return executor, nil
}
executor.Cleanup()
var buildErr *BuildError
if errors.As(err, &buildErr) {
return nil, err
} else if options.Context.Err() != nil {
return nil, b.handleError(options.Context.Err())
}
logger.SoftErrorln("Preparation failed:", err)
logger.Infoln("Will be retried in", PreparationRetryInterval, "...")
time.Sleep(PreparationRetryInterval)
}
return nil, err
}
func (b *Build) waitForTerminal(ctx context.Context, timeout time.Duration) error {
if b.Session == nil || !b.Session.Connected() {
return nil
}
timeout = b.getTerminalTimeout(ctx, timeout)
b.logger.Infoln(
fmt.Sprintf(
"Terminal is connected, will time out in %s...",
timeout.Round(time.Second),
),
)
select {
case <-ctx.Done():
err := b.Session.Kill()
if err != nil {
b.Log().WithError(err).Warn("Failed to kill session")
}
return errors.New("build cancelled, killing session")
case <-time.After(timeout):
err := fmt.Errorf(
"terminal session timed out (maximum time allowed - %s)",
timeout.Round(time.Second),
)
b.logger.Infoln(err.Error())
b.Session.TimeoutCh <- err
return err
case err := <-b.Session.DisconnectCh:
b.logger.Infoln("Terminal disconnected")
return fmt.Errorf("terminal disconnected: %w", err)
case signal := <-b.SystemInterrupt:
b.logger.Infoln("Terminal disconnected")
err := b.Session.Kill()
if err != nil {
b.Log().WithError(err).Warn("Failed to kill session")
}
return fmt.Errorf("terminal disconnected by system signal: %v", signal)
}
}
// getTerminalTimeout checks if the the job timeout comes before the
// configured terminal timeout.
func (b *Build) getTerminalTimeout(ctx context.Context, timeout time.Duration) time.Duration {
expiryTime, _ := ctx.Deadline()
if expiryTime.Before(time.Now().Add(timeout)) {
timeout = time.Until(expiryTime)
}
return timeout
}
// setTraceStatus sets the final status of a job. If the err
// is nil, the job is successful.
//
// What we send back to GitLab for a failure reason when the err
// is not nil depends:
//
// If the error can be unwrapped to `BuildError`, the BuildError's
// failure reason is given. If the failure reason is not supported
// by GitLab, it's converted to an `UnknownFailure`. If the failure
// reason is not specified, `ScriptFailure` is used.
//
// If an error cannot be unwrapped to `BuildError`, `SystemFailure`
// is used as the failure reason.
func (b *Build) setTraceStatus(trace JobTrace, err error) {
logger := b.logger.WithFields(logrus.Fields{
"duration_s": b.Duration().Seconds(),
})
if err == nil {
logger.Infoln("Job succeeded")
trace.Success()
return
}
var buildError *BuildError
if errors.As(err, &buildError) {
logger.SoftErrorln("Job failed:", err)
trace.Fail(err, JobFailureData{
Reason: b.ensureSupportedFailureReason(buildError.FailureReason),
ExitCode: buildError.ExitCode,
})
return
}
logger.Errorln("Job failed (system failure):", err)
trace.Fail(err, JobFailureData{Reason: RunnerSystemFailure})
}
func (b *Build) ensureSupportedFailureReason(reason JobFailureReason) JobFailureReason {
if reason == "" {
return ScriptFailure
}
// GitLab provides a list of supported failure reasons with the job. Should the list be empty, we use
// the minmum subset of failure reasons we know all GitLab instances support.
for _, supported := range append(
b.Features.FailureReasons,
ScriptFailure,
RunnerSystemFailure,
JobExecutionTimeout,
) {
if reason == supported {
return reason
}
}
return UnknownFailure
}
func (b *Build) setExecutorStageResolver(resolver func() ExecutorStage) {
b.statusLock.Lock()
defer b.statusLock.Unlock()
b.executorStageResolver = resolver
}
func (b *Build) CurrentExecutorStage() ExecutorStage {
b.statusLock.RLock()
defer b.statusLock.RUnlock()
if b.executorStageResolver == nil {
return ExecutorStage("")
}
return b.executorStageResolver()
}
func (b *Build) Run(globalConfig *Config, trace JobTrace) (err error) {
var executor Executor
b.logger = NewBuildLogger(trace, b.Log())
b.printRunningWithHeader()
b.setCurrentState(BuildRunStatePending)
// These defers are ordered because runBuild could panic and the recover needs to handle that panic.
// setTraceStatus needs to be last since it needs a correct error value to report the job's status
defer func() { b.setTraceStatus(trace, err) }()
defer func() {
if r := recover(); r != nil {
err = &BuildError{FailureReason: RunnerSystemFailure, Inner: fmt.Errorf("panic: %s", r)}
}
}()
defer func() { b.cleanupBuild(executor) }()
err = b.resolveSecrets()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), b.GetBuildTimeout())
defer cancel()
trace.SetCancelFunc(cancel)
trace.SetAbortFunc(cancel)
trace.SetMasked(b.GetAllVariables().Masked())
options := b.createExecutorPrepareOptions(ctx, globalConfig, trace)
provider := GetExecutorProvider(b.Runner.Executor)
if provider == nil {
return errors.New("executor not found")
}
err = provider.GetFeatures(&b.ExecutorFeatures)
if err != nil {
return fmt.Errorf("retrieving executor features: %w", err)
}
executor, err = b.executeBuildSection(executor, options, provider)
if err == nil {
err = b.run(ctx, executor)
if errWait := b.waitForTerminal(ctx, globalConfig.SessionServer.GetSessionTimeout()); errWait != nil {
b.Log().WithError(errWait).Debug("Stopped waiting for terminal")
}
}
if executor != nil {
executor.Finish(err)
}
return err
}
func (b *Build) createExecutorPrepareOptions(
ctx context.Context,
globalConfig *Config,
trace JobTrace,
) ExecutorPrepareOptions {
return ExecutorPrepareOptions{
Config: b.Runner,
Build: b,
Trace: trace,
User: globalConfig.User,
Context: ctx,
}
}
func (b *Build) resolveSecrets() error {
if b.Secrets == nil {
return nil
}
b.Secrets.expandVariables(b.GetAllVariables())
section := helpers.BuildSection{
Name: string(BuildStageResolveSecrets),
SkipMetrics: !b.JobResponse.Features.TraceSections,
Run: func() error {
resolver, err := b.secretsResolver(&b.logger, GetSecretResolverRegistry())
if err != nil {
return fmt.Errorf("creating secrets resolver: %w", err)
}
variables, err := resolver.Resolve(b.Secrets)
if err != nil {
return fmt.Errorf("resolving secrets: %w", err)
}
b.secretsVariables = variables
b.refreshAllVariables()
return nil
},
}
return section.Execute(&b.logger)
}
func (b *Build) executeBuildSection(
executor Executor,
options ExecutorPrepareOptions,
provider ExecutorProvider,
) (Executor, error) {
var err error
section := helpers.BuildSection{
Name: string(BuildStagePrepareExecutor),
SkipMetrics: !b.JobResponse.Features.TraceSections,
Run: func() error {
msg := fmt.Sprintf(
"%sPreparing the %q executor%s",
helpers.ANSI_BOLD_CYAN,
b.Runner.Executor,
helpers.ANSI_RESET,
)
b.logger.Println(msg)
executor, err = b.retryCreateExecutor(options, provider, b.logger)
return err
},
}
err = section.Execute(&b.logger)
return executor, err
}
func (b *Build) cleanupBuild(executor Executor) {
if executor != nil {
executor.Cleanup()
}
}
func (b *Build) String() string {
return helpers.ToYAML(b)
}
func (b *Build) platformAppropriatePath(s string) string {
// Check if we're dealing with a Windows path on a Windows platform
// filepath.VolumeName will return empty otherwise
if filepath.VolumeName(s) != "" {
return filepath.FromSlash(s)
}
return s
}
func (b *Build) GetDefaultVariables() JobVariables {
return JobVariables{
{
Key: "CI_BUILDS_DIR",
Value: b.platformAppropriatePath(b.RootDir),
Public: true,
Internal: true,
File: false,
},
{
Key: "CI_PROJECT_DIR",
Value: b.platformAppropriatePath(b.FullProjectDir()),
Public: true,
Internal: true,
File: false,
},
{
Key: "CI_CONCURRENT_ID",
Value: strconv.Itoa(b.RunnerID),
Public: true,
Internal: true,
File: false,
},
{
Key: "CI_CONCURRENT_PROJECT_ID",
Value: strconv.Itoa(b.ProjectRunnerID),
Public: true,
Internal: true,
File: false,
},
{
Key: "CI_SERVER",
Value: "yes",
Public: true,
Internal: true,
File: false,
},
{
Key: "CI_JOB_STATUS",
Value: string(BuildRunRuntimeRunning),
Public: true,
Internal: true,
},
}
}
func (b *Build) GetDefaultFeatureFlagsVariables() JobVariables {
variables := make(JobVariables, 0)
for _, featureFlag := range featureflags.GetAll() {
variables = append(variables, JobVariable{
Key: featureFlag.Name,
Value: strconv.FormatBool(featureFlag.DefaultValue),
Public: true,
Internal: true,
File: false,
})
}
return variables
}
func (b *Build) GetSharedEnvVariable() JobVariable {
env := JobVariable{Value: "true", Public: true, Internal: true, File: false}
if b.IsSharedEnv() {
env.Key = "CI_SHARED_ENVIRONMENT"
} else {
env.Key = "CI_DISPOSABLE_ENVIRONMENT"
}
return env
}
func (b *Build) GetTLSVariables(caFile, certFile, keyFile string) JobVariables {
variables := JobVariables{}
if b.TLSCAChain != "" {
variables = append(variables, JobVariable{
Key: caFile,
Value: b.TLSCAChain,
Public: true,
Internal: true,
File: true,
})
}
if b.TLSAuthCert != "" && b.TLSAuthKey != "" {
variables = append(
variables,
JobVariable{
Key: certFile,
Value: b.TLSAuthCert,
Public: true,
Internal: true,
File: true,
},
JobVariable{
Key: keyFile,
Value: b.TLSAuthKey,
Internal: true,
File: true,
},
)
}
return variables
}
func (b *Build) GetCITLSVariables() JobVariables {
return b.GetTLSVariables(tls.VariableCAFile, tls.VariableCertFile, tls.VariableKeyFile)
}
func (b *Build) IsSharedEnv() bool {
return b.ExecutorFeatures.Shared
}
func (b *Build) refreshAllVariables() {
b.allVariables = nil
}
func (b *Build) GetAllVariables() JobVariables {
if b.allVariables != nil {
return b.allVariables
}
variables := make(JobVariables, 0)
variables = append(variables, b.GetDefaultFeatureFlagsVariables()...)
if b.Image.Name != "" {
variables = append(
variables,
JobVariable{Key: "CI_JOB_IMAGE", Value: b.Image.Name, Public: true, Internal: true, File: false},
)
}
if b.Runner != nil {
variables = append(variables, b.Runner.GetVariables()...)
}
variables = append(variables, b.GetDefaultVariables()...)
variables = append(variables, b.GetCITLSVariables()...)
variables = append(variables, b.Variables...)
variables = append(variables, b.GetSharedEnvVariable())
variables = append(variables, AppVersion.Variables()...)
variables = append(variables, b.secretsVariables...)
b.allVariables = variables.Expand()
return b.allVariables
}
// GetRemoteURL checks if the default clone URL is overwritten by the runner
// configuration option: 'CloneURL'. If it is, we use that to create the clone
// URL.
func (b *Build) GetRemoteURL() string {
cloneURL := strings.TrimRight(b.Runner.CloneURL, "/")
if !strings.HasPrefix(cloneURL, "http") {
return b.GitInfo.RepoURL
}
variables := b.GetAllVariables()
ciJobToken := variables.Get("CI_JOB_TOKEN")
ciProjectPath := variables.Get("CI_PROJECT_PATH")
splits := strings.SplitAfterN(cloneURL, "://", 2)
return fmt.Sprintf("%sgitlab-ci-token:%s@%s/%s.git", splits[0], ciJobToken, splits[1], ciProjectPath)
}
func (b *Build) GetGitStrategy() GitStrategy {
switch b.GetAllVariables().Get("GIT_STRATEGY") {
case "clone":
return GitClone
case "fetch":
return GitFetch
case "none":
return GitNone
default:
if b.AllowGitFetch {
return GitFetch
}
return GitClone
}
}
func (b *Build) GetGitCheckout() bool {
if b.GetGitStrategy() == GitNone {
return false
}
strCheckout := b.GetAllVariables().Get("GIT_CHECKOUT")
if strCheckout == "" {
return true
}
checkout, err := strconv.ParseBool(strCheckout)
if err != nil {
return true
}
return checkout
}
func (b *Build) GetSubmoduleStrategy() SubmoduleStrategy {
if b.GetGitStrategy() == GitNone {
return SubmoduleNone
}
switch b.GetAllVariables().Get("GIT_SUBMODULE_STRATEGY") {
case "normal":
return SubmoduleNormal
case "recursive":
return SubmoduleRecursive
case "none", "":
// Default (legacy) behavior is to not update/init submodules
return SubmoduleNone
default:
// Will cause an error in AbstractShell) writeSubmoduleUpdateCmds
return SubmoduleInvalid
}
}
// GetSubmodulePaths https://git-scm.com/docs/git-submodule#Documentation/git-submodule.txt-ltpathgt82308203
func (b *Build) GetSubmodulePaths() string {
paths := b.GetAllVariables().Get("GIT_SUBMODULE_PATHS")
return paths
}
func (b *Build) GetGitCleanFlags() []string {
flags := b.GetAllVariables().Get("GIT_CLEAN_FLAGS")
if flags == "" {
flags = gitCleanFlagsDefault
}
if flags == gitCleanFlagsNone {
return []string{}
}
return strings.Fields(flags)
}
func (b *Build) GetGitFetchFlags() []string {
flags := b.GetAllVariables().Get("GIT_FETCH_EXTRA_FLAGS")
if flags == "" {
flags = gitFetchFlagsDefault
}
if flags == gitFetchFlagsNone {
return []string{}
}
return strings.Fields(flags)
}
func (b *Build) IsDebugTraceEnabled() bool {
trace, err := strconv.ParseBool(b.GetAllVariables().Get("CI_DEBUG_TRACE"))
if err != nil {
trace = false
}
if b.Runner.DebugTraceDisabled {
if trace {
b.logger.Warningln("CI_DEBUG_TRACE usage is disabled on this Runner")
}
return false
}
return trace
}
func (b *Build) GetDockerAuthConfig() string {
return b.GetAllVariables().Get("DOCKER_AUTH_CONFIG")
}
func (b *Build) GetGetSourcesAttempts() int {
retries, err := strconv.Atoi(b.GetAllVariables().Get("GET_SOURCES_ATTEMPTS"))
if err != nil {
return DefaultGetSourcesAttempts
}
return retries
}
func (b *Build) GetDownloadArtifactsAttempts() int {
retries, err := strconv.Atoi(b.GetAllVariables().Get("ARTIFACT_DOWNLOAD_ATTEMPTS"))
if err != nil {
return DefaultArtifactDownloadAttempts
}
return retries
}
func (b *Build) GetRestoreCacheAttempts() int {
retries, err := strconv.Atoi(b.GetAllVariables().Get("RESTORE_CACHE_ATTEMPTS"))
if err != nil {
return DefaultRestoreCacheAttempts
}
return retries
}
func (b *Build) GetCacheRequestTimeout() int {
timeout, err := strconv.Atoi(b.GetAllVariables().Get("CACHE_REQUEST_TIMEOUT"))
if err != nil {
return DefaultCacheRequestTimeout
}
return timeout
}
func (b *Build) GetExecutorJobSectionAttempts() (int, error) {
attempts, err := strconv.Atoi(b.GetAllVariables().Get(ExecutorJobSectionAttempts))
if err != nil {
return DefaultExecutorStageAttempts, nil
}
if validAttempts(attempts) {
return 0, &invalidAttemptError{key: ExecutorJobSectionAttempts}
}
return attempts, nil
}
func validAttempts(attempts int) bool {
return attempts < 1 || attempts > 10
}
func (b *Build) Duration() time.Duration {
return time.Since(b.createdAt)
}
func NewBuild(
jobData JobResponse,
runnerConfig *RunnerConfig,
systemInterrupt chan os.Signal,
executorData ExecutorData,
) (*Build, error) {
// Attempt to perform a deep copy of the RunnerConfig
runnerConfigCopy, err := runnerConfig.DeepCopy()
if err != nil {
return nil, fmt.Errorf("deep copy of runner config failed: %w", err)
}
return &Build{
JobResponse: jobData,
Runner: runnerConfigCopy,
SystemInterrupt: systemInterrupt,
ExecutorData: executorData,
createdAt: time.Now(),
secretsResolver: newSecretsResolver,
}, nil
}
func (b *Build) IsFeatureFlagOn(name string) bool {
if b.Runner.IsFeatureFlagDefined(name) {
return b.Runner.IsFeatureFlagOn(name)
}
return featureflags.IsOn(
b.Log().WithField("name", name),
b.GetAllVariables().Get(name),
)
}
// getFeatureFlagInfo returns the status of feature flags that differ
// from their default status.
func (b *Build) getFeatureFlagInfo() string {
var statuses []string
for _, ff := range featureflags.GetAll() {
isOn := b.IsFeatureFlagOn(ff.Name)
if isOn != ff.DefaultValue {
statuses = append(statuses, fmt.Sprintf("%s:%t", ff.Name, isOn))
}
}
return strings.Join(statuses, ", ")
}
func (b *Build) printRunningWithHeader() {
b.logger.Println("Running with", AppVersion.Line())
if b.Runner != nil && b.Runner.ShortDescription() != "" {
b.logger.Println(" on", b.Runner.Name, b.Runner.ShortDescription())
}
if featureInfo := b.getFeatureFlagInfo(); featureInfo != "" {
b.logger.Println(" feature flags:", featureInfo)
}
}
func (b *Build) IsLFSSmudgeDisabled() bool {
disabled, err := strconv.ParseBool(b.GetAllVariables().Get("GIT_LFS_SKIP_SMUDGE"))
if err != nil {
return false
}
return disabled
}
package common
import (
"fmt"
"io"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
"gitlab.com/gitlab-org/gitlab-runner/helpers/process"
url_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/url"
)
type BuildLogger struct {
log JobTrace
entry *logrus.Entry
}
func NewBuildLogger(log JobTrace, entry *logrus.Entry) BuildLogger {
return BuildLogger{
log: log,
entry: entry,
}
}
func (e *BuildLogger) WithFields(fields logrus.Fields) BuildLogger {
return NewBuildLogger(e.log, e.entry.WithFields(fields))
}
func (e *BuildLogger) SendRawLog(args ...interface{}) {
if e.log != nil {
_, _ = fmt.Fprint(e.log, args...)
}
}
func (e *BuildLogger) sendLog(logger func(args ...interface{}), logPrefix string, args ...interface{}) {
if e.log != nil {
// log lines have spaces between each argument, followed by an ANSI Reset and *then* a new-line.
//
// To achieve this, we use fmt.Sprintln and remove the newline, add the ANSI Reset and then
// append the newline again. The reason we don't use fmt.Sprint is that there's a greater
// difference between that and fmt.Sprintln than just the newline character being added
// (fmt.Sprintln consistently adds a space between arguments).
logLine := fmt.Sprintln(args...)
logLine = logLine[:len(logLine)-1]
logLine = url_helpers.ScrubSecrets(logLine)
logLine += helpers.ANSI_RESET + "\n"
e.SendRawLog(logPrefix + logLine)
if e.log.IsStdout() {
return
}
}
if len(args) == 0 {
return
}
logger(args...)
}
func (e *BuildLogger) WriterLevel(level logrus.Level) *io.PipeWriter {
return e.entry.WriterLevel(level)
}
func (e *BuildLogger) Debugln(args ...interface{}) {
if e.entry == nil {
return
}
e.entry.Debugln(args...)
}
func (e *BuildLogger) Println(args ...interface{}) {
if e.entry == nil {
return
}
e.sendLog(e.entry.Debugln, helpers.ANSI_CLEAR, args...)
}
func (e *BuildLogger) Infoln(args ...interface{}) {
if e.entry == nil {
return
}
e.sendLog(e.entry.Println, helpers.ANSI_BOLD_GREEN, args...)
}
func (e *BuildLogger) Warningln(args ...interface{}) {
if e.entry == nil {
return
}
e.sendLog(e.entry.Warningln, helpers.ANSI_YELLOW+"WARNING: ", args...)
}
func (e *BuildLogger) SoftErrorln(args ...interface{}) {
if e.entry == nil {
return
}
e.sendLog(e.entry.Warningln, helpers.ANSI_BOLD_RED+"ERROR: ", args...)
}
func (e *BuildLogger) Errorln(args ...interface{}) {
if e.entry == nil {
return
}
e.sendLog(e.entry.Errorln, helpers.ANSI_BOLD_RED+"ERROR: ", args...)
}
type ProcessLoggerAdapter struct {
buildLogger BuildLogger
}
func NewProcessLoggerAdapter(buildlogger BuildLogger) *ProcessLoggerAdapter {
return &ProcessLoggerAdapter{
buildLogger: buildlogger,
}
}
func (l *ProcessLoggerAdapter) WithFields(fields logrus.Fields) process.Logger {
l.buildLogger = l.buildLogger.WithFields(fields)
return l
}
func (l *ProcessLoggerAdapter) Warn(args ...interface{}) {
l.buildLogger.Warningln(args...)
}
package buildtest
import (
"bytes"
"errors"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
//nolint:funlen
func RunBuildWithCancel(t *testing.T, config *common.RunnerConfig, setup BuildSetupFn) {
cancelIncludeStages := []common.BuildStage{
common.BuildStagePrepare,
common.BuildStageGetSources,
}
cancelExcludeStages := []common.BuildStage{
common.BuildStageRestoreCache,
common.BuildStageDownloadArtifacts,
common.BuildStageAfterScript,
common.BuildStageArchiveOnSuccessCache,
common.BuildStageArchiveOnFailureCache,
common.BuildStageUploadOnFailureArtifacts,
common.BuildStageUploadOnSuccessArtifacts,
}
tests := map[string]struct {
onUserStep func(*common.Build, common.JobTrace)
includesStage []common.BuildStage
excludesStage []common.BuildStage
expectedErr error
}{
"system interrupt": {
onUserStep: func(build *common.Build, _ common.JobTrace) {
build.SystemInterrupt <- os.Interrupt
},
includesStage: cancelIncludeStages,
excludesStage: cancelExcludeStages,
expectedErr: &common.BuildError{FailureReason: common.RunnerSystemFailure},
},
"job is aborted": {
onUserStep: func(_ *common.Build, trace common.JobTrace) {
trace.Abort()
},
includesStage: cancelIncludeStages,
excludesStage: cancelExcludeStages,
expectedErr: &common.BuildError{FailureReason: common.JobCanceled},
},
"job is canceling": {
onUserStep: func(_ *common.Build, trace common.JobTrace) {
trace.Cancel()
},
includesStage: cancelIncludeStages,
excludesStage: cancelExcludeStages,
expectedErr: &common.BuildError{FailureReason: common.JobCanceled},
},
}
resp, err := common.GetRemoteLongRunningBuildWithAfterScript()
require.NoError(t, err)
for tn, tc := range tests {
t.Run(tn, func(t *testing.T) {
build := &common.Build{
JobResponse: resp,
Runner: config,
SystemInterrupt: make(chan os.Signal, 1),
}
buf := new(bytes.Buffer)
trace := &common.Trace{Writer: buf}
done := OnUserStage(build, func() {
tc.onUserStep(build, trace)
})
defer done()
if setup != nil {
setup(build)
}
err := RunBuildWithTrace(t, build, trace)
t.Log(buf.String())
//nolint:lll
assert.True(t, errors.Is(err, tc.expectedErr), "expected: %[1]T (%[1]v), got: %[2]T (%[2]v)", tc.expectedErr, err)
for _, stage := range tc.includesStage {
assert.Contains(t, buf.String(), common.GetStageDescription(stage))
}
for _, stage := range tc.excludesStage {
assert.NotContains(t, buf.String(), common.GetStageDescription(stage))
}
})
}
}
package buildtest
import (
"fmt"
"os"
"os/exec"
"runtime"
)
func MustBuildBinary(entrypoint string, binaryName string) string {
if runtime.GOOS == "windows" {
binaryName += ".exe"
}
cmd := exec.Command("go", "build", "-o", binaryName, entrypoint)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
fmt.Printf("Executing: %v\n", cmd)
err := cmd.Run()
if err != nil {
panic("Error on executing go build for binary: " + entrypoint)
}
return binaryName
}
package buildtest
import (
"os"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/trace"
)
func RunRemoteBuildWithJobOutputLimitExceeded(t *testing.T, config *common.RunnerConfig, setup BuildSetupFn) {
runBuildWithJobOutputLimitExceeded(t, config, setup, common.GetRemoteSuccessfulBuild)
}
func RunBuildWithJobOutputLimitExceeded(t *testing.T, config *common.RunnerConfig, setup BuildSetupFn) {
runBuildWithJobOutputLimitExceeded(t, config, setup, common.GetSuccessfulBuild)
}
type jobOutputLimitExceededTestCase struct {
jobResponse func(t *testing.T, g baseJobGetter) common.JobResponse
handleTrace func(t *testing.T, done chan struct{}, traceBuffer *trace.Buffer, trace common.JobTrace)
assertError func(t *testing.T, err error)
}
var jobOutputLimitExceededTestCases = map[string]jobOutputLimitExceededTestCase{
"successful job": {
jobResponse: func(t *testing.T, baseJobGetter baseJobGetter) common.JobResponse {
return getJobResponseWithCommands(t, baseJobGetter, "echo Hello World", "exit 0")
},
handleTrace: func(t *testing.T, done chan struct{}, traceBuffer *trace.Buffer, trace common.JobTrace) {},
assertError: func(t *testing.T, err error) {
assert.NoError(t, err)
},
},
"failed job": {
jobResponse: func(t *testing.T, baseJobGetter baseJobGetter) common.JobResponse {
return getJobResponseWithCommands(t, baseJobGetter, "echo Hello World", "exit 1")
},
handleTrace: func(t *testing.T, done chan struct{}, traceBuffer *trace.Buffer, trace common.JobTrace) {},
assertError: func(t *testing.T, err error) {
var expectedErr *common.BuildError
if assert.ErrorAs(t, err, &expectedErr) {
assert.Equal(t, 1, expectedErr.ExitCode)
assert.Empty(t, expectedErr.FailureReason)
}
},
},
"canceled job": {
jobResponse: func(t *testing.T, baseJobGetter baseJobGetter) common.JobResponse {
return getJobResponseWithCommands(t, baseJobGetter, "echo Hello World", "sleep 10", "exit 0")
},
handleTrace: func(t *testing.T, done chan struct{}, traceBuffer *trace.Buffer, trace common.JobTrace) {
for {
b, berr := traceBuffer.Bytes(0, 1024*1024)
require.NoError(t, berr)
if strings.Contains(string(b), "Job's log exceeded limit of") {
trace.Cancel()
}
select {
case <-time.After(50 * time.Millisecond):
case <-done:
return
}
}
},
assertError: func(t *testing.T, err error) {
var expectedErr *common.BuildError
if assert.ErrorAs(t, err, &expectedErr) {
assert.Equal(t, 0, expectedErr.ExitCode)
assert.Equal(t, common.JobCanceled, expectedErr.FailureReason)
}
},
},
}
// nolint:funlen
func runBuildWithJobOutputLimitExceeded(
t *testing.T,
config *common.RunnerConfig,
setup BuildSetupFn,
baseJob func() (common.JobResponse, error),
) {
for tn, tt := range jobOutputLimitExceededTestCases {
t.Run(tn, func(t *testing.T) {
build := &common.Build{
JobResponse: tt.jobResponse(t, baseJob),
Runner: config,
SystemInterrupt: make(chan os.Signal, 1),
}
if setup != nil {
setup(build)
}
runBuildWithJobOutputLimitExceededCase(t, tt, build)
})
}
}
func runBuildWithJobOutputLimitExceededCase(t *testing.T, tt jobOutputLimitExceededTestCase, build *common.Build) {
traceBuffer, err := trace.New()
require.NoError(t, err)
traceBuffer.SetLimit(12)
jobTrace := &common.Trace{Writer: traceBuffer}
done := make(chan struct{})
defer close(done)
go tt.handleTrace(t, done, traceBuffer, jobTrace)
err = RunBuildWithTrace(t, build, jobTrace)
b, berr := traceBuffer.Bytes(0, 1024*1024)
require.NoError(t, berr)
log := string(b)
assert.Contains(t, log, "Running")
assert.NotContains(t, log, "with gitlab-runner")
assert.Contains(t, log, "Job's log exceeded limit of 12 bytes.")
assert.Contains(t, log, "Job execution will continue but no more output will be collected.")
tt.assertError(t, err)
}
package buildtest
import (
"math"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/trace"
)
func RunBuildWithMasking(t *testing.T, config *common.RunnerConfig, setup BuildSetupFn) {
resp, err := common.GetRemoteSuccessfulBuildPrintVars(
config.Shell,
"MASKED_KEY",
"CLEARTEXT_KEY",
"MASKED_KEY_OTHER",
)
require.NoError(t, err)
build := &common.Build{
JobResponse: resp,
Runner: config,
}
build.Variables = append(
build.Variables,
common.JobVariable{Key: "MASKED_KEY", Value: "MASKED_VALUE", Masked: true},
common.JobVariable{Key: "CLEARTEXT_KEY", Value: "CLEARTEXT_VALUE", Masked: false},
common.JobVariable{Key: "MASKED_KEY_OTHER", Value: "MASKED_VALUE_OTHER", Masked: true},
)
if setup != nil {
setup(build)
}
buf, err := trace.New()
require.NoError(t, err)
defer buf.Close()
err = build.Run(&common.Config{}, &common.Trace{Writer: buf})
assert.NoError(t, err)
buf.Finish()
contents, err := buf.Bytes(0, math.MaxInt64)
assert.NoError(t, err)
assert.NotContains(t, string(contents), "MASKED_KEY=MASKED_VALUE")
assert.Contains(t, string(contents), "MASKED_KEY=[MASKED]")
assert.NotContains(t, string(contents), "MASKED_KEY_OTHER=MASKED_VALUE_OTHER")
assert.NotContains(t, string(contents), "MASKED_KEY_OTHER=[MASKED]_OTHER")
assert.Contains(t, string(contents), "MASKED_KEY_OTHER=[MASKED]")
assert.NotContains(t, string(contents), "CLEARTEXT_KEY=[MASKED]")
assert.Contains(t, string(contents), "CLEARTEXT_KEY=CLEARTEXT_VALUE")
}
package buildtest
import (
"bytes"
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
const testTimeout = 30 * time.Minute
type BuildSetupFn func(build *common.Build)
func RunBuildReturningOutput(t *testing.T, build *common.Build) (string, error) {
buf := new(bytes.Buffer)
err := RunBuildWithTrace(t, build, &common.Trace{Writer: buf})
output := buf.String()
t.Log(output)
return output, err
}
func RunBuildWithTrace(t *testing.T, build *common.Build, trace *common.Trace) error {
return RunBuildWithOptions(t, build, trace, &common.Config{})
}
func RunBuildWithOptions(t *testing.T, build *common.Build, trace *common.Trace, config *common.Config) error {
timeoutTimer := time.AfterFunc(testTimeout, func() {
t.Log("Timed out")
t.FailNow()
})
defer timeoutTimer.Stop()
return build.Run(config, trace)
}
func RunBuild(t *testing.T, build *common.Build) error {
err := RunBuildWithTrace(t, build, &common.Trace{Writer: os.Stdout})
return err
}
// OnStage executes the provided function when the provided stage is entered.
func OnStage(build *common.Build, stage string, fn func()) func() {
exit := make(chan struct{})
go func() {
for {
select {
case <-exit:
return
case <-time.After(200 * time.Millisecond):
currentStage := string(build.CurrentStage())
if strings.HasPrefix(currentStage, stage) {
fn()
return
}
}
}
}()
return func() {
close(exit)
}
}
// OnUserStage executes the provided function when the CurrentStage() enters
// a non-predefined stage.
func OnUserStage(build *common.Build, fn func()) func() {
return OnStage(build, "step_", fn)
}
func SetBuildFeatureFlag(build *common.Build, flag string, value bool) {
for _, v := range build.Variables {
if v.Key == flag {
v.Value = fmt.Sprint(value)
return
}
}
build.Variables = append(build.Variables, common.JobVariable{
Key: flag,
Value: fmt.Sprint(value),
})
}
type baseJobGetter func() (common.JobResponse, error)
// getJobResponseWithCommands is a wrapper that will decorate a JobResponse getter
// like common.GetRemoteSuccessfulBuild with a custom commands list
func getJobResponseWithCommands(t *testing.T, baseJobGetter baseJobGetter, commands ...string) common.JobResponse {
jobResponse, err := baseJobGetter()
require.NoError(t, err)
jobResponse.Steps[0].Script = commands
return jobResponse
}
// WithFeatureFlags runs a subtest for the on/off value for each flag provided,
// and allows a build object as part of the test to be decorated with the
// feature flag variable.
func WithEachFeatureFlag(t *testing.T, f func(t *testing.T, setup BuildSetupFn), flags ...string) {
if len(flags) == 0 {
t.Log("WithEachFeatureFlag: no feature flags provided. Running inner test with no feature flags.")
f(t, func(build *common.Build) {})
return
}
for _, flag := range flags {
for _, value := range []bool{false, true} {
t.Run(fmt.Sprintf("%v=%v", flag, value), func(t *testing.T) {
f(t, func(build *common.Build) {
SetBuildFeatureFlag(build, flag, value)
})
})
}
}
}
package common
import (
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
clihelpers "gitlab.com/ayufan/golang-cli-helpers"
)
var commands []cli.Command
type Commander interface {
Execute(c *cli.Context)
}
func RegisterCommand(command cli.Command) {
logrus.Debugln("Registering", command.Name, "command...")
commands = append(commands, command)
}
func RegisterCommand2(name, usage string, data Commander, flags ...cli.Flag) {
RegisterCommand(cli.Command{
Name: name,
Usage: usage,
Action: data.Execute,
Flags: append(flags, clihelpers.GetFlagsFromStruct(data)...),
})
}
func GetCommands() []cli.Command {
return commands
}
package common
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"math/big"
"os"
"path/filepath"
"strings"
"time"
"github.com/BurntSushi/toml"
"github.com/docker/go-units"
"github.com/sirupsen/logrus"
api "k8s.io/api/core/v1"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
"gitlab.com/gitlab-org/gitlab-runner/helpers/process"
"gitlab.com/gitlab-org/gitlab-runner/helpers/ssh"
"gitlab.com/gitlab-org/gitlab-runner/helpers/timeperiod"
"gitlab.com/gitlab-org/gitlab-runner/referees"
)
type DockerPullPolicy string
type DockerSysCtls map[string]string
const (
PullPolicyAlways = "always"
PullPolicyNever = "never"
PullPolicyIfNotPresent = "if-not-present"
DNSPolicyNone KubernetesDNSPolicy = "none"
DNSPolicyDefault KubernetesDNSPolicy = "default"
DNSPolicyClusterFirst KubernetesDNSPolicy = "cluster-first"
DNSPolicyClusterFirstWithHostNet KubernetesDNSPolicy = "cluster-first-with-host-net"
)
// InvalidTimePeriodsError represents that the time period specified is not valid.
type InvalidTimePeriodsError struct {
periods []string
cause error
}
func NewInvalidTimePeriodsError(periods []string, cause error) *InvalidTimePeriodsError {
return &InvalidTimePeriodsError{periods: periods, cause: cause}
}
func (e *InvalidTimePeriodsError) Error() string {
return fmt.Sprintf("invalid time periods %v, caused by: %v", e.periods, e.cause)
}
func (e *InvalidTimePeriodsError) Is(err error) bool {
_, ok := err.(*InvalidTimePeriodsError)
return ok
}
func (e *InvalidTimePeriodsError) Unwrap() error {
return e.cause
}
// GetPullPolicies returns a validated list of pull policies, falling back to a predefined value if empty,
// or returns an error if the list is not valid
func (c DockerConfig) GetPullPolicies() ([]DockerPullPolicy, error) {
// Default policy is always
if len(c.PullPolicy) == 0 {
return []DockerPullPolicy{PullPolicyAlways}, nil
}
// Verify pull policies
policies := make([]DockerPullPolicy, len(c.PullPolicy))
for idx, p := range c.PullPolicy {
switch p {
case PullPolicyAlways, PullPolicyIfNotPresent, PullPolicyNever:
policies[idx] = DockerPullPolicy(p)
default:
return []DockerPullPolicy{}, fmt.Errorf("unsupported docker-pull-policy: %q", p)
}
}
return policies, nil
}
// StringOrArray implements UnmarshalTOML to unmarshal either a string or array of strings.
type StringOrArray []string
func (p *StringOrArray) UnmarshalTOML(data interface{}) error {
switch v := data.(type) {
case string:
*p = StringOrArray{v}
case []interface{}:
for _, vv := range v {
switch item := vv.(type) {
case string:
*p = append(*p, item)
default:
return fmt.Errorf("unexpected data type: %v", item)
}
}
default:
return fmt.Errorf("unexpected data type: %v", v)
}
return nil
}
//nolint:lll
type DockerConfig struct {
docker.Credentials
Hostname string `toml:"hostname,omitempty" json:"hostname" long:"hostname" env:"DOCKER_HOSTNAME" description:"Custom container hostname"`
Image string `toml:"image" json:"image" long:"image" env:"DOCKER_IMAGE" description:"Docker image to be used"`
Runtime string `toml:"runtime,omitempty" json:"runtime" long:"runtime" env:"DOCKER_RUNTIME" description:"Docker runtime to be used"`
Memory string `toml:"memory,omitempty" json:"memory" long:"memory" env:"DOCKER_MEMORY" description:"Memory limit (format: <number>[<unit>]). Unit can be one of b, k, m, or g. Minimum is 4M."`
MemorySwap string `toml:"memory_swap,omitempty" json:"memory_swap" long:"memory-swap" env:"DOCKER_MEMORY_SWAP" description:"Total memory limit (memory + swap, format: <number>[<unit>]). Unit can be one of b, k, m, or g."`
MemoryReservation string `toml:"memory_reservation,omitempty" json:"memory_reservation" long:"memory-reservation" env:"DOCKER_MEMORY_RESERVATION" description:"Memory soft limit (format: <number>[<unit>]). Unit can be one of b, k, m, or g."`
CPUSetCPUs string `toml:"cpuset_cpus,omitempty" json:"cpuset_cpus" long:"cpuset-cpus" env:"DOCKER_CPUSET_CPUS" description:"String value containing the cgroups CpusetCpus to use"`
CPUS string `toml:"cpus,omitempty" json:"cpus" long:"cpus" env:"DOCKER_CPUS" description:"Number of CPUs"`
CPUShares int64 `toml:"cpu_shares,omitzero" json:"cpu_shares" long:"cpu-shares" env:"DOCKER_CPU_SHARES" description:"Number of CPU shares"`
DNS []string `toml:"dns,omitempty" json:"dns" long:"dns" env:"DOCKER_DNS" description:"A list of DNS servers for the container to use"`
DNSSearch []string `toml:"dns_search,omitempty" json:"dns_search" long:"dns-search" env:"DOCKER_DNS_SEARCH" description:"A list of DNS search domains"`
Privileged bool `toml:"privileged,omitzero" json:"privileged" long:"privileged" env:"DOCKER_PRIVILEGED" description:"Give extended privileges to container"`
DisableEntrypointOverwrite bool `toml:"disable_entrypoint_overwrite,omitzero" json:"disable_entrypoint_overwrite" long:"disable-entrypoint-overwrite" env:"DOCKER_DISABLE_ENTRYPOINT_OVERWRITE" description:"Disable the possibility for a container to overwrite the default image entrypoint"`
UsernsMode string `toml:"userns_mode,omitempty" json:"userns_mode" long:"userns" env:"DOCKER_USERNS_MODE" description:"User namespace to use"`
CapAdd []string `toml:"cap_add" json:"cap_add" long:"cap-add" env:"DOCKER_CAP_ADD" description:"Add Linux capabilities"`
CapDrop []string `toml:"cap_drop" json:"cap_drop" long:"cap-drop" env:"DOCKER_CAP_DROP" description:"Drop Linux capabilities"`
OomKillDisable bool `toml:"oom_kill_disable,omitzero" json:"oom_kill_disable" long:"oom-kill-disable" env:"DOCKER_OOM_KILL_DISABLE" description:"Do not kill processes in a container if an out-of-memory (OOM) error occurs"`
OomScoreAdjust int `toml:"oom_score_adjust,omitzero" json:"oom_score_adjust" long:"oom-score-adjust" env:"DOCKER_OOM_SCORE_ADJUST" description:"Adjust OOM score"`
SecurityOpt []string `toml:"security_opt" json:"security_opt" long:"security-opt" env:"DOCKER_SECURITY_OPT" description:"Security Options"`
Devices []string `toml:"devices" json:"devices" long:"devices" env:"DOCKER_DEVICES" description:"Add a host device to the container"`
Gpus string `toml:"gpus,omitempty" json:"gpus" long:"gpus" env:"DOCKER_GPUS" description:"Request GPUs to be used by Docker"`
DisableCache bool `toml:"disable_cache,omitzero" json:"disable_cache" long:"disable-cache" env:"DOCKER_DISABLE_CACHE" description:"Disable all container caching"`
Volumes []string `toml:"volumes,omitempty" json:"volumes" long:"volumes" env:"DOCKER_VOLUMES" description:"Bind-mount a volume and create it if it doesn't exist prior to mounting. Can be specified multiple times once per mountpoint, e.g. --docker-volumes 'test0:/test0' --docker-volumes 'test1:/test1'"`
VolumeDriver string `toml:"volume_driver,omitempty" json:"volume_driver" long:"volume-driver" env:"DOCKER_VOLUME_DRIVER" description:"Volume driver to be used"`
CacheDir string `toml:"cache_dir,omitempty" json:"cache_dir" long:"cache-dir" env:"DOCKER_CACHE_DIR" description:"Directory where to store caches"`
ExtraHosts []string `toml:"extra_hosts,omitempty" json:"extra_hosts" long:"extra-hosts" env:"DOCKER_EXTRA_HOSTS" description:"Add a custom host-to-IP mapping"`
VolumesFrom []string `toml:"volumes_from,omitempty" json:"volumes_from" long:"volumes-from" env:"DOCKER_VOLUMES_FROM" description:"A list of volumes to inherit from another container"`
NetworkMode string `toml:"network_mode,omitempty" json:"network_mode" long:"network-mode" env:"DOCKER_NETWORK_MODE" description:"Add container to a custom network"`
Links []string `toml:"links,omitempty" json:"links" long:"links" env:"DOCKER_LINKS" description:"Add link to another container"`
Services []Service `toml:"services,omitempty" json:"services" description:"Add service that is started with container"`
WaitForServicesTimeout int `toml:"wait_for_services_timeout,omitzero" json:"wait_for_services_timeout" long:"wait-for-services-timeout" env:"DOCKER_WAIT_FOR_SERVICES_TIMEOUT" description:"How long to wait for service startup"`
AllowedImages []string `toml:"allowed_images,omitempty" json:"allowed_images" long:"allowed-images" env:"DOCKER_ALLOWED_IMAGES" description:"Image allowlist"`
AllowedServices []string `toml:"allowed_services,omitempty" json:"allowed_services" long:"allowed-services" env:"DOCKER_ALLOWED_SERVICES" description:"Service allowlist"`
PullPolicy StringOrArray `toml:"pull_policy,omitempty" json:"pull_policy" long:"pull-policy" env:"DOCKER_PULL_POLICY" description:"Image pull policy: never, if-not-present, always"`
ShmSize int64 `toml:"shm_size,omitempty" json:"shm_size" long:"shm-size" env:"DOCKER_SHM_SIZE" description:"Shared memory size for docker images (in bytes)"`
Tmpfs map[string]string `toml:"tmpfs,omitempty" json:"tmpfs" long:"tmpfs" env:"DOCKER_TMPFS" description:"A toml table/json object with the format key=values. When set this will mount the specified path in the key as a tmpfs volume in the main container, using the options specified as key. For the supported options, see the documentation for the unix 'mount' command"`
ServicesTmpfs map[string]string `toml:"services_tmpfs,omitempty" json:"services_tmpfs" long:"services-tmpfs" env:"DOCKER_SERVICES_TMPFS" description:"A toml table/json object with the format key=values. When set this will mount the specified path in the key as a tmpfs volume in all the service containers, using the options specified as key. For the supported options, see the documentation for the unix 'mount' command"`
SysCtls DockerSysCtls `toml:"sysctls,omitempty" json:"sysctls" long:"sysctls" env:"DOCKER_SYSCTLS" description:"Sysctl options, a toml table/json object of key=value. Value is expected to be a string."`
HelperImage string `toml:"helper_image,omitempty" json:"helper_image" long:"helper-image" env:"DOCKER_HELPER_IMAGE" description:"[ADVANCED] Override the default helper image used to clone repos and upload artifacts"`
HelperImageFlavor string `toml:"helper_image_flavor,omitempty" json:"helper_image_flavor" long:"helper-image-flavor" env:"DOCKER_HELPER_IMAGE_FLAVOR" description:"Set helper image flavor (alpine, ubuntu), defaults to alpine"`
}
//nolint:lll
type DockerMachine struct {
MaxGrowthRate int `toml:"MaxGrowthRate,omitzero" long:"max-growth-rate" env:"MACHINE_MAX_GROWTH_RATE" description:"Maximum machines being provisioned concurrently, set to 0 for unlimited"`
IdleCount int `long:"idle-nodes" env:"MACHINE_IDLE_COUNT" description:"Maximum idle machines"`
IdleTime int `toml:"IdleTime,omitzero" long:"idle-time" env:"MACHINE_IDLE_TIME" description:"Minimum time after node can be destroyed"`
MaxBuilds int `toml:"MaxBuilds,omitzero" long:"max-builds" env:"MACHINE_MAX_BUILDS" description:"Maximum number of builds processed by machine"`
MachineDriver string `long:"machine-driver" env:"MACHINE_DRIVER" description:"The driver to use when creating machine"`
MachineName string `long:"machine-name" env:"MACHINE_NAME" description:"The template for machine name (needs to include %s)"`
MachineOptions []string `long:"machine-options" env:"MACHINE_OPTIONS" description:"Additional machine creation options"`
OffPeakPeriods []string `toml:"OffPeakPeriods,omitempty" description:"Time periods when the scheduler is in the OffPeak mode. DEPRECATED"` // DEPRECATED
OffPeakTimezone string `toml:"OffPeakTimezone,omitempty" description:"Timezone for the OffPeak periods (defaults to Local). DEPRECATED"` // DEPRECATED
OffPeakIdleCount int `toml:"OffPeakIdleCount,omitzero" description:"Maximum idle machines when the scheduler is in the OffPeak mode. DEPRECATED"` // DEPRECATED
OffPeakIdleTime int `toml:"OffPeakIdleTime,omitzero" description:"Minimum time after machine can be destroyed when the scheduler is in the OffPeak mode. DEPRECATED"` // DEPRECATED
AutoscalingConfigs []*DockerMachineAutoscaling `toml:"autoscaling" description:"Ordered list of configurations for autoscaling periods (last match wins)"`
}
//nolint:lll
type DockerMachineAutoscaling struct {
Periods []string `long:"periods" description:"List of crontab expressions for this autoscaling configuration"`
Timezone string `long:"timezone" description:"Timezone for the periods (defaults to Local)"`
IdleCount int `long:"idle-count" description:"Maximum idle machines when this configuration is active"`
IdleTime int `long:"idle-time" description:"Minimum time after which and idle machine can be destroyed when this configuration is active"`
compiledPeriods *timeperiod.TimePeriod
}
//nolint:lll
type ParallelsConfig struct {
BaseName string `toml:"base_name" json:"base_name" long:"base-name" env:"PARALLELS_BASE_NAME" description:"VM name to be used"`
TemplateName string `toml:"template_name,omitempty" json:"template_name" long:"template-name" env:"PARALLELS_TEMPLATE_NAME" description:"VM template to be created"`
DisableSnapshots bool `toml:"disable_snapshots,omitzero" json:"disable_snapshots" long:"disable-snapshots" env:"PARALLELS_DISABLE_SNAPSHOTS" description:"Disable snapshoting to speedup VM creation"`
TimeServer string `toml:"time_server,omitempty" json:"time_server" long:"time-server" env:"PARALLELS_TIME_SERVER" description:"Timeserver to sync the guests time from. Defaults to time.apple.com"`
}
//nolint:lll
type VirtualBoxConfig struct {
BaseName string `toml:"base_name" json:"base_name" long:"base-name" env:"VIRTUALBOX_BASE_NAME" description:"VM name to be used"`
BaseSnapshot string `toml:"base_snapshot,omitempty" json:"base_snapshot" long:"base-snapshot" env:"VIRTUALBOX_BASE_SNAPSHOT" description:"Name or UUID of a specific VM snapshot to clone"`
BaseFolder string `toml:"base_folder" json:"base_folder" long:"base-folder" env:"VIRTUALBOX_BASE_FOLDER" description:"Folder in which to save the new VM. If empty, uses VirtualBox default"`
DisableSnapshots bool `toml:"disable_snapshots,omitzero" json:"disable_snapshots" long:"disable-snapshots" env:"VIRTUALBOX_DISABLE_SNAPSHOTS" description:"Disable snapshoting to speedup VM creation"`
}
//nolint:lll
type CustomConfig struct {
ConfigExec string `toml:"config_exec,omitempty" json:"config_exec" long:"config-exec" env:"CUSTOM_CONFIG_EXEC" description:"Executable that allows to inject configuration values to the executor"`
ConfigArgs []string `toml:"config_args,omitempty" json:"config_args" long:"config-args" description:"Arguments for the config executable"`
ConfigExecTimeout *int `toml:"config_exec_timeout,omitempty" json:"config_exec_timeout" long:"config-exec-timeout" env:"CUSTOM_CONFIG_EXEC_TIMEOUT" description:"Timeout for the config executable (in seconds)"`
PrepareExec string `toml:"prepare_exec,omitempty" json:"prepare_exec" long:"prepare-exec" env:"CUSTOM_PREPARE_EXEC" description:"Executable that prepares executor"`
PrepareArgs []string `toml:"prepare_args,omitempty" json:"prepare_args" long:"prepare-args" description:"Arguments for the prepare executable"`
PrepareExecTimeout *int `toml:"prepare_exec_timeout,omitempty" json:"prepare_exec_timeout" long:"prepare-exec-timeout" env:"CUSTOM_PREPARE_EXEC_TIMEOUT" description:"Timeout for the prepare executable (in seconds)"`
RunExec string `toml:"run_exec" json:"run_exec" long:"run-exec" env:"CUSTOM_RUN_EXEC" description:"Executable that runs the job script in executor"`
RunArgs []string `toml:"run_args,omitempty" json:"run_args" long:"run-args" description:"Arguments for the run executable"`
CleanupExec string `toml:"cleanup_exec,omitempty" json:"cleanup_exec" long:"cleanup-exec" env:"CUSTOM_CLEANUP_EXEC" description:"Executable that cleanups after executor run"`
CleanupArgs []string `toml:"cleanup_args,omitempty" json:"cleanup_args" long:"cleanup-args" description:"Arguments for the cleanup executable"`
CleanupExecTimeout *int `toml:"cleanup_exec_timeout,omitempty" json:"cleanup_exec_timeout" long:"cleanup-exec-timeout" env:"CUSTOM_CLEANUP_EXEC_TIMEOUT" description:"Timeout for the cleanup executable (in seconds)"`
GracefulKillTimeout *int `toml:"graceful_kill_timeout,omitempty" json:"graceful_kill_timeout" long:"graceful-kill-timeout" env:"CUSTOM_GRACEFUL_KILL_TIMEOUT" description:"Graceful timeout for scripts execution after SIGTERM is sent to the process (in seconds). This limits the time given for scripts to perform the cleanup before exiting"`
ForceKillTimeout *int `toml:"force_kill_timeout,omitempty" json:"force_kill_timeout" long:"force-kill-timeout" env:"CUSTOM_FORCE_KILL_TIMEOUT" description:"Force timeout for scripts execution (in seconds). Counted from the force kill call; if process will be not terminated, Runner will abandon process termination and log an error"`
}
type KubernetesPullPolicy string
// GetPullPolicies returns a validated list of pull policies, falling back to a predefined value if empty,
// or returns an error if the list is not valid
func (c KubernetesConfig) GetPullPolicies() ([]api.PullPolicy, error) {
// Default to cluster pull policy
if len(c.PullPolicy) == 0 {
return []api.PullPolicy{""}, nil
}
// Verify pull policies
policies := make([]api.PullPolicy, len(c.PullPolicy))
for idx, p := range c.PullPolicy {
switch p {
case "":
policies[idx] = ""
case PullPolicyAlways:
policies[idx] = api.PullAlways
case PullPolicyNever:
policies[idx] = api.PullNever
case PullPolicyIfNotPresent:
policies[idx] = api.PullIfNotPresent
default:
return []api.PullPolicy{""}, fmt.Errorf("unsupported kubernetes-pull-policy: %q", p)
}
}
return policies, nil
}
type KubernetesDNSPolicy string
// Get returns one of the predefined values in kubernetes notation or an error if the value is not matched.
// If the DNSPolicy is a blank string, returns the k8s default ("ClusterFirst")
func (p KubernetesDNSPolicy) Get() (api.DNSPolicy, error) {
const defaultPolicy = api.DNSClusterFirst
switch p {
case "":
logrus.Debugf("DNSPolicy string is blank, using %q as default", defaultPolicy)
return defaultPolicy, nil
case DNSPolicyNone:
return api.DNSNone, nil
case DNSPolicyDefault:
return api.DNSDefault, nil
case DNSPolicyClusterFirst:
return api.DNSClusterFirst, nil
case DNSPolicyClusterFirstWithHostNet:
return api.DNSClusterFirstWithHostNet, nil
}
return "", fmt.Errorf("unsupported kubernetes-dns-policy: %q", p)
}
//nolint:lll
type KubernetesConfig struct {
Host string `toml:"host" json:"host" long:"host" env:"KUBERNETES_HOST" description:"Optional Kubernetes master host URL (auto-discovery attempted if not specified)"`
CertFile string `toml:"cert_file,omitempty" json:"cert_file" long:"cert-file" env:"KUBERNETES_CERT_FILE" description:"Optional Kubernetes master auth certificate"`
KeyFile string `toml:"key_file,omitempty" json:"key_file" long:"key-file" env:"KUBERNETES_KEY_FILE" description:"Optional Kubernetes master auth private key"`
CAFile string `toml:"ca_file,omitempty" json:"ca_file" long:"ca-file" env:"KUBERNETES_CA_FILE" description:"Optional Kubernetes master auth ca certificate"`
BearerTokenOverwriteAllowed bool `toml:"bearer_token_overwrite_allowed" json:"bearer_token_overwrite_allowed" long:"bearer_token_overwrite_allowed" env:"KUBERNETES_BEARER_TOKEN_OVERWRITE_ALLOWED" description:"Bool to authorize builds to specify their own bearer token for creation."`
BearerToken string `toml:"bearer_token,omitempty" json:"bearer_token" long:"bearer_token" env:"KUBERNETES_BEARER_TOKEN" description:"Optional Kubernetes service account token used to start build pods."`
Image string `toml:"image" json:"image" long:"image" env:"KUBERNETES_IMAGE" description:"Default docker image to use for builds when none is specified"`
Namespace string `toml:"namespace" json:"namespace" long:"namespace" env:"KUBERNETES_NAMESPACE" description:"Namespace to run Kubernetes jobs in"`
NamespaceOverwriteAllowed string `toml:"namespace_overwrite_allowed" json:"namespace_overwrite_allowed" long:"namespace_overwrite_allowed" env:"KUBERNETES_NAMESPACE_OVERWRITE_ALLOWED" description:"Regex to validate 'KUBERNETES_NAMESPACE_OVERWRITE' value"`
Privileged *bool `toml:"privileged,omitzero" json:"privileged" long:"privileged" env:"KUBERNETES_PRIVILEGED" description:"Run all containers with the privileged flag enabled"`
AllowPrivilegeEscalation *bool `toml:"allow_privilege_escalation,omitzero" json:"allow_privilege_escalation" long:"allow-privilege-escalation" env:"KUBERNETES_ALLOW_PRIVILEGE_ESCALATION" description:"Run all containers with the security context allowPrivilegeEscalation flag enabled. When empty, it does not define the allowPrivilegeEscalation flag in the container SecurityContext and allows Kubernetes to use the default privilege escalation behavior."`
CPULimit string `toml:"cpu_limit,omitempty" json:"cpu_limit" long:"cpu-limit" env:"KUBERNETES_CPU_LIMIT" description:"The CPU allocation given to build containers"`
CPULimitOverwriteMaxAllowed string `toml:"cpu_limit_overwrite_max_allowed,omitempty" json:"cpu_limit_overwrite_max_allowed" long:"cpu-limit-overwrite-max-allowed" env:"KUBERNETES_CPU_LIMIT_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the cpu limit can be set to. Used with the KUBERNETES_CPU_LIMIT variable in the build."`
CPURequest string `toml:"cpu_request,omitempty" json:"cpu_request" long:"cpu-request" env:"KUBERNETES_CPU_REQUEST" description:"The CPU allocation requested for build containers"`
CPURequestOverwriteMaxAllowed string `toml:"cpu_request_overwrite_max_allowed,omitempty" json:"cpu_request_overwrite_max_allowed" long:"cpu-request-overwrite-max-allowed" env:"KUBERNETES_CPU_REQUEST_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the cpu request can be set to. Used with the KUBERNETES_CPU_REQUEST variable in the build."`
MemoryLimit string `toml:"memory_limit,omitempty" json:"memory_limit" long:"memory-limit" env:"KUBERNETES_MEMORY_LIMIT" description:"The amount of memory allocated to build containers"`
MemoryLimitOverwriteMaxAllowed string `toml:"memory_limit_overwrite_max_allowed,omitempty" json:"memory_limit_overwrite_max_allowed" long:"memory-limit-overwrite-max-allowed" env:"KUBERNETES_MEMORY_LIMIT_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the memory limit can be set to. Used with the KUBERNETES_MEMORY_LIMIT variable in the build."`
MemoryRequest string `toml:"memory_request,omitempty" json:"memory_request" long:"memory-request" env:"KUBERNETES_MEMORY_REQUEST" description:"The amount of memory requested from build containers"`
MemoryRequestOverwriteMaxAllowed string `toml:"memory_request_overwrite_max_allowed,omitempty" json:"memory_request_overwrite_max_allowed" long:"memory-request-overwrite-max-allowed" env:"KUBERNETES_MEMORY_REQUEST_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the memory request can be set to. Used with the KUBERNETES_MEMORY_REQUEST variable in the build."`
EphemeralStorageLimit string `toml:"ephemeral_storage_limit,omitempty" json:"ephemeral_storage_limit" long:"ephemeral-storage-limit" env:"KUBERNETES_EPHEMERAL_STORAGE_LIMIT" description:"The amount of ephemeral storage allocated to build containers"`
EphemeralStorageLimitOverwriteMaxAllowed string `toml:"ephemeral_storage_limit_overwrite_max_allowed,omitempty" json:"ephemeral_storage_limit_overwrite_max_allowed" long:"ephemeral-storage-limit-overwrite-max-allowed" env:"KUBERNETES_EPHEMERAL_STORAGE_LIMIT_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the ephemeral limit can be set to. Used with the KUBERNETES_EPHEMERAL_STORAGE_LIMIT variable in the build."`
EphemeralStorageRequest string `toml:"ephemeral_storage_request,omitempty" json:"ephemeral_storage_request" long:"ephemeral-storage-request" env:"KUBERNETES_EPHEMERAL_STORAGE_REQUEST" description:"The amount of ephemeral storage requested from build containers"`
EphemeralStorageRequestOverwriteMaxAllowed string `toml:"ephemeral_storage_request_overwrite_max_allowed,omitempty" json:"ephemeral_storage_request_overwrite_max_allowed" long:"ephemeral-storage-request-overwrite-max-allowed" env:"KUBERNETES_EPHEMERAL_STORAGE_REQUEST_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the ephemeral storage request can be set to. Used with the KUBERNETES_EPHEMERAL_STORAGE_REQUEST variable in the build."`
ServiceCPULimit string `toml:"service_cpu_limit,omitempty" json:"service_cpu_limit" long:"service-cpu-limit" env:"KUBERNETES_SERVICE_CPU_LIMIT" description:"The CPU allocation given to build service containers"`
ServiceCPULimitOverwriteMaxAllowed string `toml:"service_cpu_limit_overwrite_max_allowed,omitempty" json:"service_cpu_limit_overwrite_max_allowed" long:"service-cpu-limit-overwrite-max-allowed" env:"KUBERNETES_SERVICE_CPU_LIMIT_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the service cpu limit can be set to. Used with the KUBERNETES_SERVICE_CPU_LIMIT variable in the build."`
ServiceCPURequest string `toml:"service_cpu_request,omitempty" json:"service_cpu_request" long:"service-cpu-request" env:"KUBERNETES_SERVICE_CPU_REQUEST" description:"The CPU allocation requested for build service containers"`
ServiceCPURequestOverwriteMaxAllowed string `toml:"service_cpu_request_overwrite_max_allowed,omitempty" json:"service_cpu_request_overwrite_max_allowed" long:"service-cpu-request-overwrite-max-allowed" env:"KUBERNETES_SERVICE_CPU_REQUEST_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the service cpu request can be set to. Used with the KUBERNETES_SERVICE_CPU_REQUEST variable in the build."`
ServiceMemoryLimit string `toml:"service_memory_limit,omitempty" json:"service_memory_limit" long:"service-memory-limit" env:"KUBERNETES_SERVICE_MEMORY_LIMIT" description:"The amount of memory allocated to build service containers"`
ServiceMemoryLimitOverwriteMaxAllowed string `toml:"service_memory_limit_overwrite_max_allowed,omitempty" json:"service_memory_limit_overwrite_max_allowed" long:"service-memory-limit-overwrite-max-allowed" env:"KUBERNETES_SERVICE_MEMORY_LIMIT_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the service memory limit can be set to. Used with the KUBERNETES_SERVICE_MEMORY_LIMIT variable in the build."`
ServiceMemoryRequest string `toml:"service_memory_request,omitempty" json:"service_memory_request" long:"service-memory-request" env:"KUBERNETES_SERVICE_MEMORY_REQUEST" description:"The amount of memory requested for build service containers"`
ServiceMemoryRequestOverwriteMaxAllowed string `toml:"service_memory_request_overwrite_max_allowed,omitempty" json:"service_memory_request_overwrite_max_allowed" long:"service-memory-request-overwrite-max-allowed" env:"KUBERNETES_SERVICE_MEMORY_REQUEST_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the service memory request can be set to. Used with the KUBERNETES_SERVICE_MEMORY_REQUEST variable in the build."`
ServiceEphemeralStorageLimit string `toml:"service_ephemeral_storage_limit,omitempty" json:"service_ephemeral_storage_limit" long:"service-ephemeral_storage-limit" env:"KUBERNETES_SERVICE_EPHEMERAL_STORAGE_LIMIT" description:"The amount of ephemeral storage allocated to build service containers"`
ServiceEphemeralStorageLimitOverwriteMaxAllowed string `toml:"service_ephemeral_storage_limit_overwrite_max_allowed,omitempty" json:"service_ephemeral_storage_limit_overwrite_max_allowed" long:"service-ephemeral_storage-limit-overwrite-max-allowed" env:"KUBERNETES_SERVICE_EPHEMERAL_STORAGE_LIMIT_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the service ephemeral storage limit can be set to. Used with the KUBERNETES_SERVICE_EPHEMERAL_STORAGE_LIMIT variable in the build."`
ServiceEphemeralStorageRequest string `toml:"service_ephemeral_storage_request,omitempty" json:"service_ephemeral_storage_request" long:"service-ephemeral_storage-request" env:"KUBERNETES_SERVICE_EPHEMERAL_STORAGE_REQUEST" description:"The amount of ephemeral storage requested for build service containers"`
ServiceEphemeralStorageRequestOverwriteMaxAllowed string `toml:"service_ephemeral_storage_request_overwrite_max_allowed,omitempty" json:"service_ephemeral_storage_request_overwrite_max_allowed" long:"service-ephemeral_storage-request-overwrite-max-allowed" env:"KUBERNETES_SERVICE_EPHEMERAL_STORAGE_REQUEST_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the service ephemeral storage request can be set to. Used with the KUBERNETES_SERVICE_EPHEMERAL_STORAGE_REQUEST variable in the build."`
HelperCPULimit string `toml:"helper_cpu_limit,omitempty" json:"helper_cpu_limit" long:"helper-cpu-limit" env:"KUBERNETES_HELPER_CPU_LIMIT" description:"The CPU allocation given to build helper containers"`
HelperCPULimitOverwriteMaxAllowed string `toml:"helper_cpu_limit_overwrite_max_allowed,omitempty" json:"helper_cpu_limit_overwrite_max_allowed" long:"helper-cpu-limit-overwrite-max-allowed" env:"KUBERNETES_HELPER_CPU_LIMIT_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the helper cpu limit can be set to. Used with the KUBERNETES_HELPER_CPU_LIMIT variable in the build."`
HelperCPURequest string `toml:"helper_cpu_request,omitempty" json:"helper_cpu_request" long:"helper-cpu-request" env:"KUBERNETES_HELPER_CPU_REQUEST" description:"The CPU allocation requested for build helper containers"`
HelperCPURequestOverwriteMaxAllowed string `toml:"helper_cpu_request_overwrite_max_allowed,omitempty" json:"helper_cpu_request_overwrite_max_allowed" long:"helper-cpu-request-overwrite-max-allowed" env:"KUBERNETES_HELPER_CPU_REQUEST_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the helper cpu request can be set to. Used with the KUBERNETES_HELPER_CPU_REQUEST variable in the build."`
HelperMemoryLimit string `toml:"helper_memory_limit,omitempty" json:"helper_memory_limit" long:"helper-memory-limit" env:"KUBERNETES_HELPER_MEMORY_LIMIT" description:"The amount of memory allocated to build helper containers"`
HelperMemoryLimitOverwriteMaxAllowed string `toml:"helper_memory_limit_overwrite_max_allowed,omitempty" json:"helper_memory_limit_overwrite_max_allowed" long:"helper-memory-limit-overwrite-max-allowed" env:"KUBERNETES_HELPER_MEMORY_LIMIT_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the helper memory limit can be set to. Used with the KUBERNETES_HELPER_MEMORY_LIMIT variable in the build."`
HelperMemoryRequest string `toml:"helper_memory_request,omitempty" json:"helper_memory_request" long:"helper-memory-request" env:"KUBERNETES_HELPER_MEMORY_REQUEST" description:"The amount of memory requested for build helper containers"`
HelperMemoryRequestOverwriteMaxAllowed string `toml:"helper_memory_request_overwrite_max_allowed,omitempty" json:"helper_memory_request_overwrite_max_allowed" long:"helper-memory-request-overwrite-max-allowed" env:"KUBERNETES_HELPER_MEMORY_REQUEST_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the helper memory request can be set to. Used with the KUBERNETES_HELPER_MEMORY_REQUEST variable in the build."`
HelperEphemeralStorageLimit string `toml:"helper_ephemeral_storage_limit,omitempty" json:"helper_ephemeral_storage_limit" long:"helper-ephemeral_storage-limit" env:"KUBERNETES_HELPER_EPHEMERAL_STORAGE_LIMIT" description:"The amount of ephemeral storage allocated to build helper containers"`
HelperEphemeralStorageLimitOverwriteMaxAllowed string `toml:"helper_ephemeral_storage_limit_overwrite_max_allowed,omitempty" json:"helper_ephemeral_storage_limit_overwrite_max_allowed" long:"helper-ephemeral_storage-limit-overwrite-max-allowed" env:"KUBERNETES_HELPER_EPHEMERAL_STORAGE_LIMIT_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the helper ephemeral storage limit can be set to. Used with the KUBERNETES_HELPER_EPHEMERAL_STORAGE_LIMIT variable in the build."`
HelperEphemeralStorageRequest string `toml:"helper_ephemeral_storage_request,omitempty" json:"helper_ephemeral_storage_request" long:"helper-ephemeral_storage-request" env:"KUBERNETES_HELPER_EPHEMERAL_STORAGE_REQUEST" description:"The amount of ephemeral storage requested for build helper containers"`
HelperEphemeralStorageRequestOverwriteMaxAllowed string `toml:"helper_ephemeral_storage_request_overwrite_max_allowed,omitempty" json:"helper_ephemeral_storage_request_overwrite_max_allowed" long:"helper-ephemeral_storage-request-overwrite-max-allowed" env:"KUBERNETES_HELPER_EPHEMERAL_STORAGE_REQUEST_OVERWRITE_MAX_ALLOWED" description:"If set, the max amount the helper ephemeral storage request can be set to. Used with the KUBERNETES_HELPER_EPHEMERAL_STORAGE_REQUEST variable in the build."`
AllowedImages []string `toml:"allowed_images,omitempty" json:"allowed_images" long:"allowed-images" env:"KUBERNETES_ALLOWED_IMAGES" description:"Image allowlist"`
AllowedServices []string `toml:"allowed_services,omitempty" json:"allowed_services" long:"allowed-services" env:"KUBERNETES_ALLOWED_SERVICES" description:"Service allowlist"`
PullPolicy StringOrArray `toml:"pull_policy,omitempty" json:"pull_policy" long:"pull-policy" env:"KUBERNETES_PULL_POLICY" description:"Policy for if/when to pull a container image (never, if-not-present, always). The cluster default will be used if not set"`
NodeSelector map[string]string `toml:"node_selector,omitempty" json:"node_selector" long:"node-selector" env:"KUBERNETES_NODE_SELECTOR" description:"A toml table/json object of key:value. Value is expected to be a string. When set this will create pods on k8s nodes that match all the key:value pairs. Only one selector is supported through environment variable configuration."`
NodeTolerations map[string]string `toml:"node_tolerations,omitempty" json:"node_tolerations" long:"node-tolerations" env:"KUBERNETES_NODE_TOLERATIONS" description:"A toml table/json object of key=value:effect. Value and effect are expected to be strings. When set, pods will tolerate the given taints. Only one toleration is supported through environment variable configuration."`
Affinity KubernetesAffinity `toml:"affinity,omitempty" json:"affinity" long:"affinity" description:"Kubernetes Affinity setting that is used to select the node that spawns a pod"`
ImagePullSecrets []string `toml:"image_pull_secrets,omitempty" json:"image_pull_secrets" long:"image-pull-secrets" env:"KUBERNETES_IMAGE_PULL_SECRETS" description:"A list of image pull secrets that are used for pulling docker image"`
HelperImage string `toml:"helper_image,omitempty" json:"helper_image" long:"helper-image" env:"KUBERNETES_HELPER_IMAGE" description:"[ADVANCED] Override the default helper image used to clone repos and upload artifacts"`
HelperImageFlavor string `toml:"helper_image_flavor,omitempty" json:"helper_image_flavor" long:"helper-image-flavor" env:"KUBERNETES_HELPER_IMAGE_FLAVOR" description:"Set helper image flavor (alpine, ubuntu), defaults to alpine"`
TerminationGracePeriodSeconds int64 `toml:"terminationGracePeriodSeconds,omitzero" json:"terminationGracePeriodSeconds" long:"terminationGracePeriodSeconds" env:"KUBERNETES_TERMINATIONGRACEPERIODSECONDS" description:"Duration after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal."`
PollInterval int `toml:"poll_interval,omitzero" json:"poll_interval" long:"poll-interval" env:"KUBERNETES_POLL_INTERVAL" description:"How frequently, in seconds, the runner will poll the Kubernetes pod it has just created to check its status"`
PollTimeout int `toml:"poll_timeout,omitzero" json:"poll_timeout" long:"poll-timeout" env:"KUBERNETES_POLL_TIMEOUT" description:"The total amount of time, in seconds, that needs to pass before the runner will timeout attempting to connect to the pod it has just created (useful for queueing more builds that the cluster can handle at a time)"`
PodLabels map[string]string `toml:"pod_labels,omitempty" json:"pod_labels" long:"pod-labels" description:"A toml table/json object of key-value. Value is expected to be a string. When set, this will create pods with the given pod labels. Environment variables will be substituted for values here."`
ServiceAccount string `toml:"service_account,omitempty" json:"service_account" long:"service-account" env:"KUBERNETES_SERVICE_ACCOUNT" description:"Executor pods will use this Service Account to talk to kubernetes API"`
ServiceAccountOverwriteAllowed string `toml:"service_account_overwrite_allowed" json:"service_account_overwrite_allowed" long:"service_account_overwrite_allowed" env:"KUBERNETES_SERVICE_ACCOUNT_OVERWRITE_ALLOWED" description:"Regex to validate 'KUBERNETES_SERVICE_ACCOUNT' value"`
PodAnnotations map[string]string `toml:"pod_annotations,omitempty" json:"pod_annotations" long:"pod-annotations" description:"A toml table/json object of key-value. Value is expected to be a string. When set, this will create pods with the given annotations. Can be overwritten in build with KUBERNETES_POD_ANNOTATION_* variables"`
PodAnnotationsOverwriteAllowed string `toml:"pod_annotations_overwrite_allowed" json:"pod_annotations_overwrite_allowed" long:"pod_annotations_overwrite_allowed" env:"KUBERNETES_POD_ANNOTATIONS_OVERWRITE_ALLOWED" description:"Regex to validate 'KUBERNETES_POD_ANNOTATIONS_*' values"`
PodSecurityContext KubernetesPodSecurityContext `toml:"pod_security_context,omitempty" namespace:"pod-security-context" description:"A security context attached to each build pod"`
Volumes KubernetesVolumes `toml:"volumes"`
HostAliases []KubernetesHostAliases `toml:"host_aliases,omitempty" json:"host_aliases" long:"host_aliases" description:"Add a custom host-to-IP mapping"`
Services []Service `toml:"services,omitempty" json:"services" description:"Add service that is started with container"`
CapAdd []string `toml:"cap_add" json:"cap_add" long:"cap-add" env:"KUBERNETES_CAP_ADD" description:"Add Linux capabilities"`
CapDrop []string `toml:"cap_drop" json:"cap_drop" long:"cap-drop" env:"KUBERNETES_CAP_DROP" description:"Drop Linux capabilities"`
DNSPolicy KubernetesDNSPolicy `toml:"dns_policy,omitempty" json:"dns_policy" long:"dns-policy" env:"KUBERNETES_DNS_POLICY" description:"How Kubernetes should try to resolve DNS from the created pods. If unset, Kubernetes will use the default 'ClusterFirst'. Valid values are: none, default, cluster-first, cluster-first-with-host-net"`
DNSConfig KubernetesDNSConfig `toml:"dns_config" json:"dns_config" description:"Pod DNS config"`
}
//nolint:lll
type KubernetesDNSConfig struct {
Nameservers []string `toml:"nameservers" description:"A list of IP addresses that will be used as DNS servers for the Pod."`
Options []KubernetesDNSConfigOption `toml:"options" description:"An optional list of objects where each object may have a name property (required) and a value property (optional)."`
Searches []string `toml:"searches" description:"A list of DNS search domains for hostname lookup in the Pod."`
}
type KubernetesDNSConfigOption struct {
Name string `toml:"name"`
Value *string `toml:"value,omitempty"`
}
type KubernetesVolumes struct {
HostPaths []KubernetesHostPath `toml:"host_path" description:"The host paths which will be mounted"`
PVCs []KubernetesPVC `toml:"pvc" description:"The persistent volume claims that will be mounted"`
ConfigMaps []KubernetesConfigMap `toml:"config_map" description:"The config maps which will be mounted as volumes"`
Secrets []KubernetesSecret `toml:"secret" description:"The secret maps which will be mounted"`
EmptyDirs []KubernetesEmptyDir `toml:"empty_dir" description:"The empty dirs which will be mounted"`
CSIs []KubernetesCSI `toml:"csi" description:"The CSI volumes which will be mounted"`
}
//nolint:lll
type KubernetesConfigMap struct {
Name string `toml:"name" json:"name" description:"The name of the volume and ConfigMap to use"`
MountPath string `toml:"mount_path" description:"Path where volume should be mounted inside of container"`
SubPath string `toml:"sub_path,omitempty" description:"The sub-path of the volume to mount (defaults to volume root)"`
ReadOnly bool `toml:"read_only,omitempty" description:"If this volume should be mounted read only"`
Items map[string]string `toml:"items,omitempty" description:"Key-to-path mapping for keys from the config map that should be used."`
}
//nolint:lll
type KubernetesHostPath struct {
Name string `toml:"name" json:"name" description:"The name of the volume"`
MountPath string `toml:"mount_path" description:"Path where volume should be mounted inside of container"`
SubPath string `toml:"sub_path,omitempty" description:"The sub-path of the volume to mount (defaults to volume root)"`
ReadOnly bool `toml:"read_only,omitempty" description:"If this volume should be mounted read only"`
HostPath string `toml:"host_path,omitempty" description:"Path from the host that should be mounted as a volume"`
}
//nolint:lll
type KubernetesPVC struct {
Name string `toml:"name" json:"name" description:"The name of the volume and PVC to use"`
MountPath string `toml:"mount_path" description:"Path where volume should be mounted inside of container"`
SubPath string `toml:"sub_path,omitempty" description:"The sub-path of the volume to mount (defaults to volume root)"`
ReadOnly bool `toml:"read_only,omitempty" description:"If this volume should be mounted read only"`
}
//nolint:lll
type KubernetesSecret struct {
Name string `toml:"name" json:"name" description:"The name of the volume and Secret to use"`
MountPath string `toml:"mount_path" description:"Path where volume should be mounted inside of container"`
SubPath string `toml:"sub_path,omitempty" description:"The sub-path of the volume to mount (defaults to volume root)"`
ReadOnly bool `toml:"read_only,omitempty" description:"If this volume should be mounted read only"`
Items map[string]string `toml:"items,omitempty" description:"Key-to-path mapping for keys from the secret that should be used."`
}
//nolint:lll
type KubernetesEmptyDir struct {
Name string `toml:"name" json:"name" description:"The name of the volume and EmptyDir to use"`
MountPath string `toml:"mount_path" description:"Path where volume should be mounted inside of container"`
SubPath string `toml:"sub_path,omitempty" description:"The sub-path of the volume to mount (defaults to volume root)"`
Medium string `toml:"medium,omitempty" description:"Set to 'Memory' to have a tmpfs"`
}
//nolint:lll
type KubernetesCSI struct {
Name string `toml:"name" json:"name" description:"The name of the CSI volume and volumeMount to use"`
MountPath string `toml:"mount_path" description:"Path where volume should be mounted inside of container"`
SubPath string `toml:"sub_path,omitempty" description:"The sub-path of the volume to mount (defaults to volume root)"`
Driver string `toml:"driver" description:"A string value that specifies the name of the volume driver to use."`
FSType string `toml:"fs_type" description:"Filesystem type to mount. If not provided, the empty value is passed to the associated CSI driver which will determine the default filesystem to apply."`
ReadOnly bool `toml:"read_only,omitempty" description:"If this volume should be mounted read only"`
VolumeAttributes map[string]string `toml:"volume_attributes,omitempty" description:"Key-value pair mapping for attributes of the CSI volume."`
}
//nolint:lll
type KubernetesPodSecurityContext struct {
FSGroup *int64 `toml:"fs_group,omitempty" long:"fs-group" env:"KUBERNETES_POD_SECURITY_CONTEXT_FS_GROUP" description:"A special supplemental group that applies to all containers in a pod"`
RunAsGroup *int64 `toml:"run_as_group,omitempty" long:"run-as-group" env:"KUBERNETES_POD_SECURITY_CONTEXT_RUN_AS_GROUP" description:"The GID to run the entrypoint of the container process"`
RunAsNonRoot *bool `toml:"run_as_non_root,omitempty" long:"run-as-non-root" env:"KUBERNETES_POD_SECURITY_CONTEXT_RUN_AS_NON_ROOT" description:"Indicates that the container must run as a non-root user"`
RunAsUser *int64 `toml:"run_as_user,omitempty" long:"run-as-user" env:"KUBERNETES_POD_SECURITY_CONTEXT_RUN_AS_USER" description:"The UID to run the entrypoint of the container process"`
SupplementalGroups []int64 `toml:"supplemental_groups,omitempty" long:"supplemental-groups" description:"A list of groups applied to the first process run in each container, in addition to the container's primary GID"`
}
//nolint:lll
type KubernetesAffinity struct {
NodeAffinity *KubernetesNodeAffinity `toml:"node_affinity,omitempty" json:"node_affinity" long:"node-affinity" description:"Node affinity is conceptually similar to nodeSelector -- it allows you to constrain which nodes your pod is eligible to be scheduled on, based on labels on the node."`
}
//nolint:lll
type KubernetesNodeAffinity struct {
RequiredDuringSchedulingIgnoredDuringExecution *NodeSelector `toml:"required_during_scheduling_ignored_during_execution,omitempty" json:"required_during_scheduling_ignored_during_execution"`
PreferredDuringSchedulingIgnoredDuringExecution []PreferredSchedulingTerm `toml:"preferred_during_scheduling_ignored_during_execution,omitempty" json:"preferred_during_scheduling_ignored_during_execution"`
}
//nolint:lll
type KubernetesHostAliases struct {
IP string `toml:"ip" json:"ip" long:"ip" description:"The IP address you want to attach hosts to"`
Hostnames []string `toml:"hostnames" json:"hostnames" long:"hostnames" description:"A list of hostnames that will be attached to the IP"`
}
type NodeSelector struct {
NodeSelectorTerms []NodeSelectorTerm `toml:"node_selector_terms" json:"node_selector_terms"`
}
type PreferredSchedulingTerm struct {
Weight int32 `toml:"weight" json:"weight"`
Preference NodeSelectorTerm `toml:"preference" json:"preference"`
}
type NodeSelectorTerm struct {
MatchExpressions []NodeSelectorRequirement `toml:"match_expressions,omitempty" json:"match_expressions"`
MatchFields []NodeSelectorRequirement `toml:"match_fields,omitempty" json:"match_fields"`
}
//nolint:lll
type NodeSelectorRequirement struct {
Key string `toml:"key,omitempty" json:"key"`
Operator string `toml:"operator,omitempty" json:"operator"`
Values []string `toml:"values,omitempty" json:"values"`
}
//nolint:lll
type Service struct {
Name string `toml:"name" long:"name" description:"The image path for the service"`
Alias string `toml:"alias,omitempty" long:"alias" description:"The alias of the service"`
Command []string `toml:"command" long:"command" description:"Command or script that should be used as the container’s command. Syntax is similar to https://docs.docker.com/engine/reference/builder/#cmd"`
Entrypoint []string `toml:"entrypoint" long:"entrypoint" description:"Command or script that should be executed as the container’s entrypoint. syntax is similar to https://docs.docker.com/engine/reference/builder/#entrypoint"`
}
func (s *Service) ToImageDefinition() Image {
return Image{
Name: s.Name,
Alias: s.Alias,
Command: s.Command,
Entrypoint: s.Entrypoint,
}
}
//nolint:lll
type RunnerCredentials struct {
URL string `toml:"url" json:"url" short:"u" long:"url" env:"CI_SERVER_URL" required:"true" description:"Runner URL"`
Token string `toml:"token" json:"token" short:"t" long:"token" env:"CI_SERVER_TOKEN" required:"true" description:"Runner token"`
TLSCAFile string `toml:"tls-ca-file,omitempty" json:"tls-ca-file" long:"tls-ca-file" env:"CI_SERVER_TLS_CA_FILE" description:"File containing the certificates to verify the peer when using HTTPS"`
TLSCertFile string `toml:"tls-cert-file,omitempty" json:"tls-cert-file" long:"tls-cert-file" env:"CI_SERVER_TLS_CERT_FILE" description:"File containing certificate for TLS client auth when using HTTPS"`
TLSKeyFile string `toml:"tls-key-file,omitempty" json:"tls-key-file" long:"tls-key-file" env:"CI_SERVER_TLS_KEY_FILE" description:"File containing private key for TLS client auth when using HTTPS"`
}
//nolint:lll
type CacheGCSCredentials struct {
AccessID string `toml:"AccessID,omitempty" long:"access-id" env:"CACHE_GCS_ACCESS_ID" description:"ID of GCP Service Account used to access the storage"`
PrivateKey string `toml:"PrivateKey,omitempty" long:"private-key" env:"CACHE_GCS_PRIVATE_KEY" description:"Private key used to sign GCS requests"`
}
//nolint:lll
type CacheGCSConfig struct {
CacheGCSCredentials
CredentialsFile string `toml:"CredentialsFile,omitempty" long:"credentials-file" env:"GOOGLE_APPLICATION_CREDENTIALS" description:"File with GCP credentials, containing AccessID and PrivateKey"`
BucketName string `toml:"BucketName,omitempty" long:"bucket-name" env:"CACHE_GCS_BUCKET_NAME" description:"Name of the bucket where cache will be stored"`
}
//nolint:lll
type CacheS3Config struct {
ServerAddress string `toml:"ServerAddress,omitempty" long:"server-address" env:"CACHE_S3_SERVER_ADDRESS" description:"A host:port to the used S3-compatible server"`
AccessKey string `toml:"AccessKey,omitempty" long:"access-key" env:"CACHE_S3_ACCESS_KEY" description:"S3 Access Key"`
SecretKey string `toml:"SecretKey,omitempty" long:"secret-key" env:"CACHE_S3_SECRET_KEY" description:"S3 Secret Key"`
BucketName string `toml:"BucketName,omitempty" long:"bucket-name" env:"CACHE_S3_BUCKET_NAME" description:"Name of the bucket where cache will be stored"`
BucketLocation string `toml:"BucketLocation,omitempty" long:"bucket-location" env:"CACHE_S3_BUCKET_LOCATION" description:"Name of S3 region"`
Insecure bool `toml:"Insecure,omitempty" long:"insecure" env:"CACHE_S3_INSECURE" description:"Use insecure mode (without https)"`
}
//nolint:lll
type CacheAzureCredentials struct {
AccountName string `toml:"AccountName,omitempty" long:"account-name" env:"CACHE_AZURE_ACCOUNT_NAME" description:"Account name for Azure Blob Storage"`
AccountKey string `toml:"AccountKey,omitempty" long:"account-key" env:"CACHE_AZURE_ACCOUNT_KEY" description:"Access key for Azure Blob Storage"`
}
//nolint:lll
type CacheAzureConfig struct {
CacheAzureCredentials
ContainerName string `toml:"ContainerName,omitempty" long:"container-name" env:"CACHE_AZURE_CONTAINER_NAME" description:"Name of the Azure container where cache will be stored"`
StorageDomain string `toml:"StorageDomain,omitempty" long:"storage-domain" env:"CACHE_AZURE_STORAGE_DOMAIN" description:"Domain name of the Azure storage (e.g. blob.core.windows.net)"`
}
//nolint:lll
type CacheConfig struct {
Type string `toml:"Type,omitempty" long:"type" env:"CACHE_TYPE" description:"Select caching method"`
Path string `toml:"Path,omitempty" long:"path" env:"CACHE_PATH" description:"Name of the path to prepend to the cache URL"`
Shared bool `toml:"Shared,omitempty" long:"shared" env:"CACHE_SHARED" description:"Enable cache sharing between runners."`
S3 *CacheS3Config `toml:"s3,omitempty" json:"s3" namespace:"s3"`
GCS *CacheGCSConfig `toml:"gcs,omitempty" json:"gcs" namespace:"gcs"`
Azure *CacheAzureConfig `toml:"azure,omitempty" json:"azure" namespace:"azure"`
}
//nolint:lll
type RunnerSettings struct {
Executor string `toml:"executor" json:"executor" long:"executor" env:"RUNNER_EXECUTOR" required:"true" description:"Select executor, eg. shell, docker, etc."`
BuildsDir string `toml:"builds_dir,omitempty" json:"builds_dir" long:"builds-dir" env:"RUNNER_BUILDS_DIR" description:"Directory where builds are stored"`
CacheDir string `toml:"cache_dir,omitempty" json:"cache_dir" long:"cache-dir" env:"RUNNER_CACHE_DIR" description:"Directory where build cache is stored"`
CloneURL string `toml:"clone_url,omitempty" json:"clone_url" long:"clone-url" env:"CLONE_URL" description:"Overwrite the default URL used to clone or fetch the git ref"`
Environment []string `toml:"environment,omitempty" json:"environment" long:"env" env:"RUNNER_ENV" description:"Custom environment variables injected to build environment"`
PreCloneScript string `toml:"pre_clone_script,omitempty" json:"pre_clone_script" long:"pre-clone-script" env:"RUNNER_PRE_CLONE_SCRIPT" description:"Runner-specific command script executed before code is pulled"`
PreBuildScript string `toml:"pre_build_script,omitempty" json:"pre_build_script" long:"pre-build-script" env:"RUNNER_PRE_BUILD_SCRIPT" description:"Runner-specific command script executed after code is pulled, just before build executes"`
PostBuildScript string `toml:"post_build_script,omitempty" json:"post_build_script" long:"post-build-script" env:"RUNNER_POST_BUILD_SCRIPT" description:"Runner-specific command script executed after code is pulled and just after build executes"`
DebugTraceDisabled bool `toml:"debug_trace_disabled,omitempty" json:"debug_trace_disabled" long:"debug-trace-disabled" env:"RUNNER_DEBUG_TRACE_DISABLED" description:"When set to true Runner will disable the possibility of using the CI_DEBUG_TRACE feature"`
Shell string `toml:"shell,omitempty" json:"shell" long:"shell" env:"RUNNER_SHELL" description:"Select bash, cmd, pwsh or powershell"`
CustomBuildDir *CustomBuildDir `toml:"custom_build_dir,omitempty" json:"custom_build_dir" group:"custom build dir configuration" namespace:"custom_build_dir"`
Referees *referees.Config `toml:"referees,omitempty" json:"referees" group:"referees configuration" namespace:"referees"`
Cache *CacheConfig `toml:"cache,omitempty" json:"cache" group:"cache configuration" namespace:"cache"`
// GracefulKillTimeout and ForceKillTimeout aren't exposed to the users yet
// because not every executor supports it. We also have to keep in mind that
// the CustomConfig has its configuration fields for termination so when
// every executor supports graceful termination we should expose this single
// configuration for all executors.
GracefulKillTimeout *int `toml:"-"`
ForceKillTimeout *int `toml:"-"`
FeatureFlags map[string]bool `toml:"feature_flags" json:"feature_flags" long:"feature-flags" env:"FEATURE_FLAGS" description:"Enable/Disable feature flags https://docs.gitlab.com/runner/configuration/feature-flags.html"`
SSH *ssh.Config `toml:"ssh,omitempty" json:"ssh" group:"ssh executor" namespace:"ssh"`
Docker *DockerConfig `toml:"docker,omitempty" json:"docker" group:"docker executor" namespace:"docker"`
Parallels *ParallelsConfig `toml:"parallels,omitempty" json:"parallels" group:"parallels executor" namespace:"parallels"`
VirtualBox *VirtualBoxConfig `toml:"virtualbox,omitempty" json:"virtualbox" group:"virtualbox executor" namespace:"virtualbox"`
Machine *DockerMachine `toml:"machine,omitempty" json:"machine" group:"docker machine provider" namespace:"machine"`
Kubernetes *KubernetesConfig `toml:"kubernetes,omitempty" json:"kubernetes" group:"kubernetes executor" namespace:"kubernetes"`
Custom *CustomConfig `toml:"custom,omitempty" json:"custom" group:"custom executor" namespace:"custom"`
}
//nolint:lll
type RunnerConfig struct {
Name string `toml:"name" json:"name" short:"name" long:"description" env:"RUNNER_NAME" description:"Runner name"`
Limit int `toml:"limit,omitzero" json:"limit" long:"limit" env:"RUNNER_LIMIT" description:"Maximum number of builds processed by this runner"`
OutputLimit int `toml:"output_limit,omitzero" long:"output-limit" env:"RUNNER_OUTPUT_LIMIT" description:"Maximum build trace size in kilobytes"`
RequestConcurrency int `toml:"request_concurrency,omitzero" long:"request-concurrency" env:"RUNNER_REQUEST_CONCURRENCY" description:"Maximum concurrency for job requests"`
RunnerCredentials
RunnerSettings
}
//nolint:lll
type SessionServer struct {
ListenAddress string `toml:"listen_address,omitempty" json:"listen_address" description:"Address that the runner will communicate directly with"`
AdvertiseAddress string `toml:"advertise_address,omitempty" json:"advertise_address" description:"Address the runner will expose to the world to connect to the session server"`
SessionTimeout int `toml:"session_timeout,omitempty" json:"session_timeout" description:"How long a terminal session can be active after a build completes, in seconds"`
}
//nolint:lll
type Config struct {
ListenAddress string `toml:"listen_address,omitempty" json:"listen_address"`
SessionServer SessionServer `toml:"session_server,omitempty" json:"session_server"`
Concurrent int `toml:"concurrent" json:"concurrent"`
CheckInterval int `toml:"check_interval" json:"check_interval" description:"Define active checking interval of jobs"`
LogLevel *string `toml:"log_level" json:"log_level" description:"Define log level (one of: panic, fatal, error, warning, info, debug)"`
LogFormat *string `toml:"log_format" json:"log_format" description:"Define log format (one of: runner, text, json)"`
User string `toml:"user,omitempty" json:"user"`
Runners []*RunnerConfig `toml:"runners" json:"runners"`
SentryDSN *string `toml:"sentry_dsn"`
ModTime time.Time `toml:"-"`
Loaded bool `toml:"-"`
}
//nolint:lll
type CustomBuildDir struct {
Enabled bool `toml:"enabled,omitempty" json:"enabled" long:"enabled" env:"CUSTOM_BUILD_DIR_ENABLED" description:"Enable job specific build directories"`
}
func (c *CacheS3Config) ShouldUseIAMCredentials() bool {
return c.ServerAddress == "" || c.AccessKey == "" || c.SecretKey == ""
}
func (c *CacheConfig) GetPath() string {
return c.Path
}
func (c *CacheConfig) GetShared() bool {
return c.Shared
}
func (r *RunnerSettings) GetGracefulKillTimeout() time.Duration {
return getDuration(r.GracefulKillTimeout, process.GracefulTimeout)
}
func (r *RunnerSettings) GetForceKillTimeout() time.Duration {
return getDuration(r.ForceKillTimeout, process.KillTimeout)
}
// IsFeatureFlagOn check if the specified feature flag is on. If the feature
// flag is not configured it will return the default value.
func (r *RunnerSettings) IsFeatureFlagOn(name string) bool {
if r.IsFeatureFlagDefined(name) {
return r.FeatureFlags[name]
}
for _, ff := range featureflags.GetAll() {
if ff.Name == name {
return ff.DefaultValue
}
}
return false
}
// IsFeatureFlagDefined checks if the feature flag is defined in the runner
// configuration.
func (r *RunnerSettings) IsFeatureFlagDefined(name string) bool {
_, ok := r.FeatureFlags[name]
return ok
}
func getDuration(source *int, defaultValue time.Duration) time.Duration {
if source == nil {
return defaultValue
}
timeout := *source
if timeout <= 0 {
return defaultValue
}
return time.Duration(timeout) * time.Second
}
func (c *SessionServer) GetSessionTimeout() time.Duration {
if c.SessionTimeout > 0 {
return time.Duration(c.SessionTimeout) * time.Second
}
return DefaultSessionTimeout
}
func (c *DockerConfig) GetNanoCPUs() (int64, error) {
if c.CPUS == "" {
return 0, nil
}
cpu, ok := new(big.Rat).SetString(c.CPUS)
if !ok {
return 0, fmt.Errorf("failed to parse %v as a rational number", c.CPUS)
}
nano, _ := cpu.Mul(cpu, big.NewRat(1e9, 1)).Float64()
return int64(nano), nil
}
func (c *DockerConfig) getMemoryBytes(size string, fieldName string) int64 {
if size == "" {
return 0
}
bytes, err := units.RAMInBytes(size)
if err != nil {
logrus.Fatalf("Error parsing docker %s: %s", fieldName, err)
}
return bytes
}
func (c *DockerConfig) GetMemory() int64 {
return c.getMemoryBytes(c.Memory, "memory")
}
func (c *DockerConfig) GetMemorySwap() int64 {
return c.getMemoryBytes(c.MemorySwap, "memory_swap")
}
func (c *DockerConfig) GetMemoryReservation() int64 {
return c.getMemoryBytes(c.MemoryReservation, "memory_reservation")
}
func (c *DockerConfig) GetOomKillDisable() *bool {
return &c.OomKillDisable
}
func (c *KubernetesConfig) GetPollAttempts() int {
if c.PollTimeout <= 0 {
c.PollTimeout = KubernetesPollTimeout
}
return c.PollTimeout / c.GetPollInterval()
}
func (c *KubernetesConfig) GetPollInterval() int {
if c.PollInterval <= 0 {
c.PollInterval = KubernetesPollInterval
}
return c.PollInterval
}
func (c *KubernetesConfig) GetNodeTolerations() []api.Toleration {
var tolerations []api.Toleration
for toleration, effect := range c.NodeTolerations {
newToleration := api.Toleration{
Effect: api.TaintEffect(effect),
}
if strings.Contains(toleration, "=") {
parts := strings.Split(toleration, "=")
newToleration.Key = parts[0]
if len(parts) > 1 {
newToleration.Value = parts[1]
}
newToleration.Operator = api.TolerationOpEqual
} else {
newToleration.Key = toleration
newToleration.Operator = api.TolerationOpExists
}
tolerations = append(tolerations, newToleration)
}
return tolerations
}
func (c *KubernetesConfig) GetPodSecurityContext() *api.PodSecurityContext {
podSecurityContext := c.PodSecurityContext
if podSecurityContext.FSGroup == nil &&
podSecurityContext.RunAsGroup == nil &&
podSecurityContext.RunAsNonRoot == nil &&
podSecurityContext.RunAsUser == nil &&
len(podSecurityContext.SupplementalGroups) == 0 {
return nil
}
return &api.PodSecurityContext{
FSGroup: podSecurityContext.FSGroup,
RunAsGroup: podSecurityContext.RunAsGroup,
RunAsNonRoot: podSecurityContext.RunAsNonRoot,
RunAsUser: podSecurityContext.RunAsUser,
SupplementalGroups: podSecurityContext.SupplementalGroups,
}
}
func (c *KubernetesConfig) GetAffinity() *api.Affinity {
var affinity api.Affinity
if c.Affinity.NodeAffinity != nil {
affinity.NodeAffinity = c.GetNodeAffinity()
}
return &affinity
}
func (c *KubernetesConfig) GetDNSConfig() *api.PodDNSConfig {
if len(c.DNSConfig.Nameservers) == 0 && len(c.DNSConfig.Searches) == 0 && len(c.DNSConfig.Options) == 0 {
return nil
}
var config api.PodDNSConfig
config.Nameservers = c.DNSConfig.Nameservers
config.Searches = c.DNSConfig.Searches
for _, opt := range c.DNSConfig.Options {
config.Options = append(config.Options, api.PodDNSConfigOption{
Name: opt.Name,
Value: opt.Value,
})
}
return &config
}
//nolint:lll
func (c *KubernetesConfig) GetNodeAffinity() *api.NodeAffinity {
var nodeAffinity api.NodeAffinity
if c.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = c.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.GetNodeSelector()
}
for _, preferred := range c.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution {
nodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append(nodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution, preferred.GetPreferredSchedulingTerm())
}
return &nodeAffinity
}
func (c *NodeSelector) GetNodeSelector() *api.NodeSelector {
var nodeSelector api.NodeSelector
for _, selector := range c.NodeSelectorTerms {
nodeSelector.NodeSelectorTerms = append(nodeSelector.NodeSelectorTerms, selector.GetNodeSelectorTerm())
}
return &nodeSelector
}
func (c *NodeSelectorRequirement) GetNodeSelectorRequirement() api.NodeSelectorRequirement {
return api.NodeSelectorRequirement{
Key: c.Key,
Operator: api.NodeSelectorOperator(c.Operator),
Values: c.Values,
}
}
//nolint:lll
func (c *NodeSelectorTerm) GetNodeSelectorTerm() api.NodeSelectorTerm {
var nodeSelectorTerm = api.NodeSelectorTerm{}
for _, expression := range c.MatchExpressions {
nodeSelectorTerm.MatchExpressions = append(nodeSelectorTerm.MatchExpressions, expression.GetNodeSelectorRequirement())
}
for _, fields := range c.MatchFields {
nodeSelectorTerm.MatchFields = append(nodeSelectorTerm.MatchFields, fields.GetNodeSelectorRequirement())
}
return nodeSelectorTerm
}
func (c *PreferredSchedulingTerm) GetPreferredSchedulingTerm() api.PreferredSchedulingTerm {
return api.PreferredSchedulingTerm{
Weight: c.Weight,
Preference: c.Preference.GetNodeSelectorTerm(),
}
}
func (c *KubernetesConfig) GetHostAliases() []api.HostAlias {
var hostAliases []api.HostAlias
for _, hostAlias := range c.HostAliases {
hostAliases = append(
hostAliases,
api.HostAlias{
IP: hostAlias.IP,
Hostnames: hostAlias.Hostnames,
},
)
}
return hostAliases
}
func (c *DockerMachine) GetIdleCount() int {
autoscaling := c.getActiveAutoscalingConfig()
if autoscaling != nil {
return autoscaling.IdleCount
}
return c.IdleCount
}
func (c *DockerMachine) GetIdleTime() int {
autoscaling := c.getActiveAutoscalingConfig()
if autoscaling != nil {
return autoscaling.IdleTime
}
return c.IdleTime
}
// getActiveAutoscalingConfig returns the autoscaling config matching the current time.
// It goes through the [[docker.machine.autoscaling]] entries and returns the last one to match.
// Returns nil on no matching entries.
func (c *DockerMachine) getActiveAutoscalingConfig() *DockerMachineAutoscaling {
var activeConf *DockerMachineAutoscaling
for _, conf := range c.AutoscalingConfigs {
if conf.compiledPeriods.InPeriod() {
activeConf = conf
}
}
return activeConf
}
func (c *DockerMachine) CompilePeriods() error {
var err error
for _, a := range c.AutoscalingConfigs {
err = a.compilePeriods()
if err != nil {
return err
}
}
return nil
}
var periodTimer = time.Now
func (a *DockerMachineAutoscaling) compilePeriods() error {
periods, err := timeperiod.TimePeriodsWithTimer(a.Periods, a.Timezone, periodTimer)
if err != nil {
return NewInvalidTimePeriodsError(a.Periods, err)
}
a.compiledPeriods = periods
return nil
}
func (c *DockerMachine) logDeprecationWarning() {
if len(c.OffPeakPeriods) != 0 {
logrus.Warning("OffPeak docker machine configuration is deprecated and has been removed since 14.0. " +
"Please convert the setting into a [[docker.machine.autoscaling]] configuration instead: " +
"https://docs.gitlab.com/runner/configuration/autoscale.html#off-peak-time-mode-configuration-deprecated")
}
}
func (c *RunnerCredentials) GetURL() string {
return c.URL
}
func (c *RunnerCredentials) GetTLSCAFile() string {
return c.TLSCAFile
}
func (c *RunnerCredentials) GetTLSCertFile() string {
return c.TLSCertFile
}
func (c *RunnerCredentials) GetTLSKeyFile() string {
return c.TLSKeyFile
}
func (c *RunnerCredentials) GetToken() string {
return c.Token
}
func (c *RunnerCredentials) ShortDescription() string {
return helpers.ShortenToken(c.Token)
}
func (c *RunnerCredentials) UniqueID() string {
return c.URL + c.Token
}
func (c *RunnerCredentials) Log() *logrus.Entry {
if c.ShortDescription() != "" {
return logrus.WithField("runner", c.ShortDescription())
}
return logrus.WithFields(logrus.Fields{})
}
func (c *RunnerCredentials) SameAs(other *RunnerCredentials) bool {
return c.URL == other.URL && c.Token == other.Token
}
func (c *RunnerConfig) String() string {
return fmt.Sprintf("%v url=%v token=%v executor=%v", c.Name, c.URL, c.Token, c.Executor)
}
func (c *RunnerConfig) GetRequestConcurrency() int {
if c.RequestConcurrency <= 0 {
return 1
}
return c.RequestConcurrency
}
func (c *RunnerConfig) GetVariables() JobVariables {
variables := JobVariables{
{Key: "CI_RUNNER_SHORT_TOKEN", Value: c.ShortDescription(), Public: true, Internal: true, File: false},
}
for _, environment := range c.Environment {
if variable, err := ParseVariable(environment); err == nil {
variable.Internal = true
variables = append(variables, variable)
}
}
return variables
}
// DeepCopy attempts to make a deep clone of the object
func (c *RunnerConfig) DeepCopy() (*RunnerConfig, error) {
var r RunnerConfig
bytes, err := json.Marshal(c)
if err != nil {
return nil, fmt.Errorf("serialization of runner config failed: %w", err)
}
err = json.Unmarshal(bytes, &r)
if err != nil {
return nil, fmt.Errorf("deserialization of runner config failed: %w", err)
}
return &r, err
}
func NewConfig() *Config {
return &Config{
Concurrent: 1,
SessionServer: SessionServer{
SessionTimeout: int(DefaultSessionTimeout.Seconds()),
},
}
}
func (c *Config) StatConfig(configFile string) error {
_, err := os.Stat(configFile)
if err != nil {
return err
}
return nil
}
func (c *Config) LoadConfig(configFile string) error {
info, err := os.Stat(configFile)
// permission denied is soft error
if os.IsNotExist(err) {
return nil
} else if err != nil {
return err
}
if _, err = toml.DecodeFile(configFile, c); err != nil {
return err
}
for _, runner := range c.Runners {
if runner.Machine == nil {
continue
}
err := runner.Machine.CompilePeriods()
if err != nil {
return err
}
runner.Machine.logDeprecationWarning()
}
c.ModTime = info.ModTime()
c.Loaded = true
return nil
}
func (c *Config) SaveConfig(configFile string) error {
var newConfig bytes.Buffer
newBuffer := bufio.NewWriter(&newConfig)
if err := toml.NewEncoder(newBuffer).Encode(c); err != nil {
logrus.Fatalf("Error encoding TOML: %s", err)
return err
}
if err := newBuffer.Flush(); err != nil {
return err
}
// create directory to store configuration
err := os.MkdirAll(filepath.Dir(configFile), 0700)
if err != nil {
return err
}
// write config file
if err := ioutil.WriteFile(configFile, newConfig.Bytes(), 0600); err != nil {
return err
}
c.Loaded = true
return nil
}
func (c *Config) GetCheckInterval() time.Duration {
if c.CheckInterval > 0 {
return time.Duration(c.CheckInterval) * time.Second
}
return CheckInterval
}
package common
import (
"context"
"errors"
"fmt"
"github.com/sirupsen/logrus"
)
// ExecutorData is an empty interface representing free-form data
// executor will use. Meant to be casted, e.g. virtual machine details.
type ExecutorData interface{}
// ExecutorCommand stores the script executor will run on a given stage.
// If Predefined it will try to use already allocated resources.
type ExecutorCommand struct {
Script string
Stage BuildStage
Predefined bool
Context context.Context
}
// ExecutorStage represents a stage of build execution in the executor scope.
type ExecutorStage string
const (
// ExecutorStageCreated means the executor is being initialized, i.e. created.
ExecutorStageCreated ExecutorStage = "created"
// ExecutorStagePrepare means the executor is preparing its environment, initializing dependencies.
ExecutorStagePrepare ExecutorStage = "prepare"
// ExecutorStageFinish means the executor has finished build execution.
ExecutorStageFinish ExecutorStage = "finish"
// ExecutorStageCleanup means the executor is cleaning up resources.
ExecutorStageCleanup ExecutorStage = "cleanup"
)
// ExecutorPrepareOptions stores any data necessary for the executor to prepare
// the environment for running a build. This includes runner configuration, build data, etc.
type ExecutorPrepareOptions struct {
Config *RunnerConfig
Build *Build
Trace JobTrace
User string
Context context.Context
}
// Executor represents entities responsible for build execution.
// It prepares the environment, runs the build and cleans up resources.
// See more in https://docs.gitlab.com/runner/executors/
type Executor interface {
// Shell returns data about the shell and scripts this executor is bound to.
Shell() *ShellScriptInfo
// Prepare prepares the environment for build execution. e.g. connects to SSH, creates containers.
Prepare(options ExecutorPrepareOptions) error
// Run executes a command on the prepared environment.
Run(cmd ExecutorCommand) error
// Finish marks the build execution as finished.
Finish(err error)
// Cleanup cleans any resources left by build execution.
Cleanup()
// GetCurrentStage returns current stage of build execution.
GetCurrentStage() ExecutorStage
// SetCurrentStage sets the current stage of build execution.
SetCurrentStage(stage ExecutorStage)
}
// ExecutorProvider is responsible for managing the lifetime of executors, acquiring resources,
// retrieving executor metadata, etc.
type ExecutorProvider interface {
// CanCreate returns whether the executor provider has the necessary data to create an executor.
CanCreate() bool
// Create creates a new executor. No resource allocation happens.
Create() Executor
// Acquire acquires the necessary resources for the executor to run, e.g. finds a virtual machine.
Acquire(config *RunnerConfig) (ExecutorData, error)
// Release releases any resources locked by Acquire.
Release(config *RunnerConfig, data ExecutorData)
// GetFeatures returns metadata about the features the executor supports, e.g. variables, services, shell.
GetFeatures(features *FeaturesInfo) error
// GetConfigInfo extracts metadata about the config the executor is using, e.g. GPUs.
GetConfigInfo(input *RunnerConfig, output *ConfigInfo)
// GetDefaultShell returns the name of the default shell for the executor.
GetDefaultShell() string
}
// BuildError represents an error during build execution, not related to
// the job script, e.g. failed to create container, establish ssh connection.
type BuildError struct {
Inner error
FailureReason JobFailureReason
ExitCode int
}
// Error implements the error interface.
func (b *BuildError) Error() string {
if b.Inner == nil {
return "error"
}
return b.Inner.Error()
}
func (b *BuildError) Is(err error) bool {
buildErr, ok := err.(*BuildError)
if !ok {
return false
}
return buildErr.FailureReason == b.FailureReason
}
func (b *BuildError) Unwrap() error {
return b.Inner
}
// MakeBuildError returns an new instance of BuildError.
func MakeBuildError(format string, args ...interface{}) error {
return &BuildError{
Inner: fmt.Errorf(format, args...),
}
}
var executorProviders map[string]ExecutorProvider
func validateExecutorProvider(provider ExecutorProvider) error {
if provider.GetDefaultShell() == "" {
return errors.New("default shell not implemented")
}
if !provider.CanCreate() {
return errors.New("cannot create executor")
}
if err := provider.GetFeatures(&FeaturesInfo{}); err != nil {
return fmt.Errorf("cannot get features: %w", err)
}
return nil
}
// RegisterExecutorProvider maps an ExecutorProvider to an executor name, i.e. registers it.
func RegisterExecutorProvider(executor string, provider ExecutorProvider) {
logrus.Debugln("Registering", executor, "executor...")
if err := validateExecutorProvider(provider); err != nil {
panic("Executor cannot be registered: " + err.Error())
}
if executorProviders == nil {
executorProviders = make(map[string]ExecutorProvider)
}
if _, ok := executorProviders[executor]; ok {
panic("Executor already exist: " + executor)
}
executorProviders[executor] = provider
}
// GetExecutorProvider returns an ExecutorProvider by name from the registered ones.
func GetExecutorProvider(executor string) ExecutorProvider {
if executorProviders == nil {
return nil
}
provider := executorProviders[executor]
return provider
}
// GetExecutorNames returns a list of all registered executor names.
func GetExecutorNames() []string {
var names []string
for name := range executorProviders {
names = append(names, name)
}
return names
}
// GetExecutorProviders returns a list of all registered executor providers.
func GetExecutorProviders() []ExecutorProvider {
var providers []ExecutorProvider
for _, executorProvider := range executorProviders {
providers = append(providers, executorProvider)
}
return providers
}
func NewExecutor(executor string) Executor {
provider := GetExecutorProvider(executor)
if provider != nil {
return provider.Create()
}
return nil
}
package common
import (
"context"
"fmt"
"io"
"strings"
"time"
url_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/url"
"gitlab.com/gitlab-org/gitlab-runner/helpers/vault/auth_methods"
)
type UpdateState int
type PatchState int
type UploadState int
type DownloadState int
type JobState string
type JobFailureReason string
const (
Pending JobState = "pending"
Running JobState = "running"
Failed JobState = "failed"
Success JobState = "success"
)
const (
ScriptFailure JobFailureReason = "script_failure"
RunnerSystemFailure JobFailureReason = "runner_system_failure"
JobExecutionTimeout JobFailureReason = "job_execution_timeout"
UnknownFailure JobFailureReason = "unknown_failure"
// JobCanceled is only internal to runner, and not used inside of rails.
JobCanceled JobFailureReason = "job_canceled"
)
const (
UpdateSucceeded UpdateState = iota
UpdateAcceptedButNotCompleted
UpdateTraceValidationFailed
UpdateNotFound
UpdateAbort
UpdateFailed
)
const (
PatchSucceeded PatchState = iota
PatchNotFound
PatchAbort
PatchRangeMismatch
PatchFailed
)
const (
UploadSucceeded UploadState = iota
UploadTooLarge
UploadForbidden
UploadFailed
UploadServiceUnavailable
)
const (
DownloadSucceeded DownloadState = iota
DownloadForbidden
DownloadFailed
DownloadNotFound
)
type FeaturesInfo struct {
Variables bool `json:"variables"`
Image bool `json:"image"`
Services bool `json:"services"`
Artifacts bool `json:"artifacts"`
Cache bool `json:"cache"`
Shared bool `json:"shared"`
UploadMultipleArtifacts bool `json:"upload_multiple_artifacts"`
UploadRawArtifacts bool `json:"upload_raw_artifacts"`
Session bool `json:"session"`
Terminal bool `json:"terminal"`
Refspecs bool `json:"refspecs"`
Masking bool `json:"masking"`
Proxy bool `json:"proxy"`
RawVariables bool `json:"raw_variables"`
ArtifactsExclude bool `json:"artifacts_exclude"`
MultiBuildSteps bool `json:"multi_build_steps"`
TraceReset bool `json:"trace_reset"`
TraceChecksum bool `json:"trace_checksum"`
TraceSize bool `json:"trace_size"`
VaultSecrets bool `json:"vault_secrets"`
Cancelable bool `json:"cancelable"`
ReturnExitCode bool `json:"return_exit_code"`
}
type ConfigInfo struct {
Gpus string `json:"gpus"`
}
type RegisterRunnerParameters struct {
Description string `json:"description,omitempty"`
Tags string `json:"tag_list,omitempty"`
RunUntagged bool `json:"run_untagged"`
Locked bool `json:"locked"`
AccessLevel string `json:"access_level,omitempty"`
MaximumTimeout int `json:"maximum_timeout,omitempty"`
Active bool `json:"active"`
}
type RegisterRunnerRequest struct {
RegisterRunnerParameters
Info VersionInfo `json:"info,omitempty"`
Token string `json:"token,omitempty"`
}
type RegisterRunnerResponse struct {
Token string `json:"token,omitempty"`
}
type VerifyRunnerRequest struct {
Token string `json:"token,omitempty"`
}
type UnregisterRunnerRequest struct {
Token string `json:"token,omitempty"`
}
type VersionInfo struct {
Name string `json:"name,omitempty"`
Version string `json:"version,omitempty"`
Revision string `json:"revision,omitempty"`
Platform string `json:"platform,omitempty"`
Architecture string `json:"architecture,omitempty"`
Executor string `json:"executor,omitempty"`
Shell string `json:"shell,omitempty"`
Features FeaturesInfo `json:"features"`
Config ConfigInfo `json:"config,omitempty"`
}
type JobRequest struct {
Info VersionInfo `json:"info,omitempty"`
Token string `json:"token,omitempty"`
LastUpdate string `json:"last_update,omitempty"`
Session *SessionInfo `json:"session,omitempty"`
}
type SessionInfo struct {
URL string `json:"url,omitempty"`
Certificate string `json:"certificate,omitempty"`
Authorization string `json:"authorization,omitempty"`
}
type JobInfo struct {
Name string `json:"name"`
Stage string `json:"stage"`
ProjectID int `json:"project_id"`
ProjectName string `json:"project_name"`
}
type GitInfoRefType string
const (
RefTypeBranch GitInfoRefType = "branch"
RefTypeTag GitInfoRefType = "tag"
)
type GitInfo struct {
RepoURL string `json:"repo_url"`
Ref string `json:"ref"`
Sha string `json:"sha"`
BeforeSha string `json:"before_sha"`
RefType GitInfoRefType `json:"ref_type"`
Refspecs []string `json:"refspecs"`
Depth int `json:"depth"`
}
type RunnerInfo struct {
Timeout int `json:"timeout"`
}
type StepScript []string
type StepName string
const (
StepNameScript StepName = "script"
StepNameAfterScript StepName = "after_script"
)
type StepWhen string
const (
StepWhenOnFailure StepWhen = "on_failure"
StepWhenOnSuccess StepWhen = "on_success"
StepWhenAlways StepWhen = "always"
)
type CachePolicy string
const (
CachePolicyUndefined CachePolicy = ""
CachePolicyPullPush CachePolicy = "pull-push"
CachePolicyPull CachePolicy = "pull"
CachePolicyPush CachePolicy = "push"
)
type Step struct {
Name StepName `json:"name"`
Script StepScript `json:"script"`
Timeout int `json:"timeout"`
When StepWhen `json:"when"`
AllowFailure bool `json:"allow_failure"`
}
type Steps []Step
type Image struct {
Name string `json:"name"`
Alias string `json:"alias,omitempty"`
Command []string `json:"command,omitempty"`
Entrypoint []string `json:"entrypoint,omitempty"`
Ports []Port `json:"ports,omitempty"`
}
type Port struct {
Number int `json:"number,omitempty"`
Protocol string `json:"protocol,omitempty"`
Name string `json:"name,omitempty"`
}
type Services []Image
type ArtifactPaths []string
type ArtifactExclude []string
type ArtifactWhen string
const (
ArtifactWhenOnFailure ArtifactWhen = "on_failure"
ArtifactWhenOnSuccess ArtifactWhen = "on_success"
ArtifactWhenAlways ArtifactWhen = "always"
)
func (when ArtifactWhen) OnSuccess() bool {
return when == "" || when == ArtifactWhenOnSuccess || when == ArtifactWhenAlways
}
func (when ArtifactWhen) OnFailure() bool {
return when == ArtifactWhenOnFailure || when == ArtifactWhenAlways
}
type ArtifactFormat string
const (
ArtifactFormatDefault ArtifactFormat = ""
ArtifactFormatZip ArtifactFormat = "zip"
ArtifactFormatGzip ArtifactFormat = "gzip"
ArtifactFormatRaw ArtifactFormat = "raw"
)
type Artifact struct {
Name string `json:"name"`
Untracked bool `json:"untracked"`
Paths ArtifactPaths `json:"paths"`
Exclude ArtifactExclude `json:"exclude"`
When ArtifactWhen `json:"when"`
Type string `json:"artifact_type"`
Format ArtifactFormat `json:"artifact_format"`
ExpireIn string `json:"expire_in"`
}
type Artifacts []Artifact
type Cache struct {
Key string `json:"key"`
Untracked bool `json:"untracked"`
Policy CachePolicy `json:"policy"`
Paths ArtifactPaths `json:"paths"`
When CacheWhen `json:"when"`
}
type CacheWhen string
const (
CacheWhenOnFailure CacheWhen = "on_failure"
CacheWhenOnSuccess CacheWhen = "on_success"
CacheWhenAlways CacheWhen = "always"
)
func (when CacheWhen) ShouldCache(jobSuccess bool) bool {
if jobSuccess {
return when.OnSuccess()
}
return when.OnFailure()
}
func (when CacheWhen) OnSuccess() bool {
return when == "" || when == CacheWhenOnSuccess || when == CacheWhenAlways
}
func (when CacheWhen) OnFailure() bool {
return when == CacheWhenOnFailure || when == CacheWhenAlways
}
func (c Cache) CheckPolicy(wanted CachePolicy) (bool, error) {
switch c.Policy {
case CachePolicyUndefined, CachePolicyPullPush:
return true, nil
case CachePolicyPull, CachePolicyPush:
return wanted == c.Policy, nil
}
return false, fmt.Errorf("unknown cache policy %s", c.Policy)
}
type Caches []Cache
type Credentials struct {
Type string `json:"type"`
URL string `json:"url"`
Username string `json:"username"`
Password string `json:"password"`
}
type DependencyArtifactsFile struct {
Filename string `json:"filename"`
Size int64 `json:"size"`
}
type Dependency struct {
ID int `json:"id"`
Token string `json:"token"`
Name string `json:"name"`
ArtifactsFile DependencyArtifactsFile `json:"artifacts_file"`
}
type Dependencies []Dependency
type GitlabFeatures struct {
TraceSections bool `json:"trace_sections"`
FailureReasons []JobFailureReason `json:"failure_reasons"`
}
type JobResponse struct {
ID int `json:"id"`
Token string `json:"token"`
AllowGitFetch bool `json:"allow_git_fetch"`
JobInfo JobInfo `json:"job_info"`
GitInfo GitInfo `json:"git_info"`
RunnerInfo RunnerInfo `json:"runner_info"`
Variables JobVariables `json:"variables"`
Steps Steps `json:"steps"`
Image Image `json:"image"`
Services Services `json:"services"`
Artifacts Artifacts `json:"artifacts"`
Cache Caches `json:"cache"`
Credentials []Credentials `json:"credentials"`
Dependencies Dependencies `json:"dependencies"`
Features GitlabFeatures `json:"features"`
Secrets Secrets `json:"secrets,omitempty"`
TLSCAChain string `json:"-"`
TLSAuthCert string `json:"-"`
TLSAuthKey string `json:"-"`
}
type Secrets map[string]Secret
type Secret struct {
Vault *VaultSecret `json:"vault,omitempty"`
File *bool `json:"file,omitempty"`
}
type VaultSecret struct {
Server VaultServer `json:"server"`
Engine VaultEngine `json:"engine"`
Path string `json:"path"`
Field string `json:"field"`
}
type VaultServer struct {
URL string `json:"url"`
Auth VaultAuth `json:"auth"`
}
type VaultAuth struct {
Name string `json:"name"`
Path string `json:"path"`
Data VaultAuthData `json:"data"`
}
type VaultAuthData map[string]interface{}
type VaultEngine struct {
Name string `json:"name"`
Path string `json:"path"`
}
func (s Secrets) expandVariables(vars JobVariables) {
for _, secret := range s {
secret.expandVariables(vars)
}
}
func (s Secret) expandVariables(vars JobVariables) {
if s.Vault != nil {
s.Vault.expandVariables(vars)
}
}
// IsFile defines whether the variable should be of type FILE or no.
//
// The default behavior is to represent the variable as FILE type.
// If defined by the user - set to whatever was chosen.
func (s Secret) IsFile() bool {
if s.File == nil {
return SecretVariableDefaultsToFile
}
return *s.File
}
func (s *VaultSecret) expandVariables(vars JobVariables) {
s.Server.expandVariables(vars)
s.Engine.expandVariables(vars)
s.Path = vars.ExpandValue(s.Path)
s.Field = vars.ExpandValue(s.Field)
}
func (s *VaultSecret) AuthName() string {
return s.Server.Auth.Name
}
func (s *VaultSecret) AuthPath() string {
return s.Server.Auth.Path
}
func (s *VaultSecret) AuthData() auth_methods.Data {
return auth_methods.Data(s.Server.Auth.Data)
}
func (s *VaultSecret) EngineName() string {
return s.Engine.Name
}
func (s *VaultSecret) EnginePath() string {
return s.Engine.Path
}
func (s *VaultSecret) SecretPath() string {
return s.Path
}
func (s *VaultSecret) SecretField() string {
return s.Field
}
func (s *VaultServer) expandVariables(vars JobVariables) {
s.URL = vars.ExpandValue(s.URL)
s.Auth.expandVariables(vars)
}
func (a *VaultAuth) expandVariables(vars JobVariables) {
a.Name = vars.ExpandValue(a.Name)
a.Path = vars.ExpandValue(a.Path)
for field, value := range a.Data {
a.Data[field] = vars.ExpandValue(fmt.Sprintf("%s", value))
}
}
func (e *VaultEngine) expandVariables(vars JobVariables) {
e.Name = vars.ExpandValue(e.Name)
e.Path = vars.ExpandValue(e.Path)
}
func (j *JobResponse) RepoCleanURL() string {
return url_helpers.CleanURL(j.GitInfo.RepoURL)
}
func (j *JobResponse) JobURL() string {
url := strings.TrimSuffix(j.RepoCleanURL(), ".git")
return fmt.Sprintf("%s/-/jobs/%d", url, j.ID)
}
type UpdateJobRequest struct {
Info VersionInfo `json:"info,omitempty"`
Token string `json:"token,omitempty"`
State JobState `json:"state,omitempty"`
FailureReason JobFailureReason `json:"failure_reason,omitempty"`
Checksum string `json:"checksum,omitempty"` // deprecated
Output JobTraceOutput `json:"output,omitempty"`
ExitCode int `json:"exit_code,omitempty"`
}
type JobTraceOutput struct {
Checksum string `json:"checksum,omitempty"`
Bytesize int `json:"bytesize,omitempty"`
}
//nolint:lll
type JobCredentials struct {
ID int `long:"id" env:"CI_JOB_ID" description:"The build ID to download and upload artifacts for"`
Token string `long:"token" env:"CI_JOB_TOKEN" required:"true" description:"Build token"`
URL string `long:"url" env:"CI_SERVER_URL" required:"true" description:"GitLab CI URL"`
TLSCAFile string `long:"tls-ca-file" env:"CI_SERVER_TLS_CA_FILE" description:"File containing the certificates to verify the peer when using HTTPS"`
TLSCertFile string `long:"tls-cert-file" env:"CI_SERVER_TLS_CERT_FILE" description:"File containing certificate for TLS client auth with runner when using HTTPS"`
TLSKeyFile string `long:"tls-key-file" env:"CI_SERVER_TLS_KEY_FILE" description:"File containing private key for TLS client auth with runner when using HTTPS"`
}
func (j *JobCredentials) GetURL() string {
return j.URL
}
func (j *JobCredentials) GetTLSCAFile() string {
return j.TLSCAFile
}
func (j *JobCredentials) GetTLSCertFile() string {
return j.TLSCertFile
}
func (j *JobCredentials) GetTLSKeyFile() string {
return j.TLSKeyFile
}
func (j *JobCredentials) GetToken() string {
return j.Token
}
type UpdateJobInfo struct {
ID int
State JobState
FailureReason JobFailureReason
Output JobTraceOutput
ExitCode int
}
type ArtifactsOptions struct {
BaseName string
ExpireIn string
Format ArtifactFormat
Type string
}
type FailuresCollector interface {
RecordFailure(reason JobFailureReason, runnerDescription string)
}
type JobTrace interface {
io.Writer
Success()
Fail(err error, failureData JobFailureData)
SetCancelFunc(cancelFunc context.CancelFunc)
Cancel() bool
SetAbortFunc(abortFunc context.CancelFunc)
Abort() bool
SetFailuresCollector(fc FailuresCollector)
SetMasked(values []string)
IsStdout() bool
}
type UpdateJobResult struct {
State UpdateState
CancelRequested bool
NewUpdateInterval time.Duration
}
type PatchTraceResult struct {
SentOffset int
CancelRequested bool
State PatchState
NewUpdateInterval time.Duration
}
func NewPatchTraceResult(sentOffset int, state PatchState, newUpdateInterval int) PatchTraceResult {
return PatchTraceResult{
SentOffset: sentOffset,
State: state,
NewUpdateInterval: time.Duration(newUpdateInterval) * time.Second,
}
}
type Network interface {
RegisterRunner(config RunnerCredentials, parameters RegisterRunnerParameters) *RegisterRunnerResponse
VerifyRunner(config RunnerCredentials) bool
UnregisterRunner(config RunnerCredentials) bool
RequestJob(ctx context.Context, config RunnerConfig, sessionInfo *SessionInfo) (*JobResponse, bool)
UpdateJob(config RunnerConfig, jobCredentials *JobCredentials, jobInfo UpdateJobInfo) UpdateJobResult
PatchTrace(config RunnerConfig, jobCredentials *JobCredentials, content []byte, startOffset int) PatchTraceResult
DownloadArtifacts(config JobCredentials, artifactsFile io.WriteCloser, directDownload *bool) DownloadState
UploadRawArtifacts(config JobCredentials, reader io.ReadCloser, options ArtifactsOptions) UploadState
ProcessJob(config RunnerConfig, buildCredentials *JobCredentials) (JobTrace, error)
}
package common
import (
"errors"
"fmt"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
)
type logger interface {
Println(args ...interface{})
Warningln(args ...interface{})
}
type SecretsResolver interface {
Resolve(secrets Secrets) (JobVariables, error)
}
type SecretResolverRegistry interface {
Register(f secretResolverFactory)
GetFor(secret Secret) (SecretResolver, error)
}
type secretResolverFactory func(secret Secret) SecretResolver
type SecretResolver interface {
Name() string
IsSupported() bool
Resolve() (string, error)
}
var (
secretResolverRegistry = new(defaultSecretResolverRegistry)
ErrMissingLogger = errors.New("logger not provided")
ErrMissingSecretResolver = errors.New("no resolver that can handle the secret")
)
func GetSecretResolverRegistry() SecretResolverRegistry {
return secretResolverRegistry
}
type defaultSecretResolverRegistry struct {
factories []secretResolverFactory
}
func (r *defaultSecretResolverRegistry) Register(f secretResolverFactory) {
r.factories = append(r.factories, f)
}
func (r *defaultSecretResolverRegistry) GetFor(secret Secret) (SecretResolver, error) {
for _, f := range r.factories {
sr := f(secret)
if sr.IsSupported() {
return sr, nil
}
}
return nil, ErrMissingSecretResolver
}
func newSecretsResolver(l logger, registry SecretResolverRegistry) (SecretsResolver, error) {
if l == nil {
return nil, ErrMissingLogger
}
sr := &defaultSecretsResolver{
logger: l,
secretResolverRegistry: registry,
}
return sr, nil
}
type defaultSecretsResolver struct {
logger logger
secretResolverRegistry SecretResolverRegistry
}
func (r *defaultSecretsResolver) Resolve(secrets Secrets) (JobVariables, error) {
if secrets == nil {
return nil, nil
}
msg := fmt.Sprintf(
"%sResolving secrets%s",
helpers.ANSI_BOLD_CYAN,
helpers.ANSI_RESET,
)
r.logger.Println(msg)
variables := make(JobVariables, 0)
for variableKey, secret := range secrets {
r.logger.Println(fmt.Sprintf("Resolving secret %q...", variableKey))
v, err := r.handleSecret(variableKey, secret)
if err != nil {
return nil, err
}
if v != nil {
variables = append(variables, *v)
}
}
return variables, nil
}
func (r *defaultSecretsResolver) handleSecret(variableKey string, secret Secret) (*JobVariable, error) {
sr, err := r.secretResolverRegistry.GetFor(secret)
if err != nil {
r.logger.Warningln(fmt.Sprintf("Not resolved: %v", err))
return nil, nil
}
r.logger.Println(fmt.Sprintf("Using %q secret resolver...", sr.Name()))
value, err := sr.Resolve()
if err != nil {
return nil, err
}
variable := &JobVariable{
Key: variableKey,
Value: value,
File: secret.IsFile(),
}
return variable, nil
}
package common
import (
"fmt"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
)
type ShellConfiguration struct {
Environment []string
DockerCommand []string
Command string
Arguments []string
PassFile bool
Extension string
}
type ShellType int
const (
NormalShell ShellType = iota
LoginShell
)
func (s *ShellConfiguration) GetCommandWithArguments() []string {
parts := []string{s.Command}
parts = append(parts, s.Arguments...)
return parts
}
func (s *ShellConfiguration) String() string {
return helpers.ToYAML(s)
}
type ShellScriptInfo struct {
Shell string
Build *Build
Type ShellType
User string
RunnerCommand string
PreCloneScript string
PreBuildScript string
PostBuildScript string
}
type Shell interface {
GetName() string
GetFeatures(features *FeaturesInfo)
IsDefault() bool
GetConfiguration(info ShellScriptInfo) (*ShellConfiguration, error)
GenerateScript(buildStage BuildStage, info ShellScriptInfo) (string, error)
}
var shells map[string]Shell
func RegisterShell(shell Shell) {
logrus.Debugln("Registering", shell.GetName(), "shell...")
if shells == nil {
shells = make(map[string]Shell)
}
if shells[shell.GetName()] != nil {
panic("Shell already exist: " + shell.GetName())
}
shells[shell.GetName()] = shell
}
func GetShell(shell string) Shell {
if shells == nil {
return nil
}
return shells[shell]
}
func GetShellConfiguration(info ShellScriptInfo) (*ShellConfiguration, error) {
shell := GetShell(info.Shell)
if shell == nil {
return nil, fmt.Errorf("shell %s not found", info.Shell)
}
return shell.GetConfiguration(info)
}
func GenerateShellScript(buildStage BuildStage, info ShellScriptInfo) (string, error) {
shell := GetShell(info.Shell)
if shell == nil {
return "", fmt.Errorf("shell %s not found", info.Shell)
}
return shell.GenerateScript(buildStage, info)
}
func GetDefaultShell() string {
if shells == nil {
panic("no shells defined")
}
for _, shell := range shells {
if shell.IsDefault() {
return shell.GetName()
}
}
panic("no default shell defined")
}
package common
import (
"bytes"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"fmt"
"math/big"
"net/http"
"os"
"path"
"runtime"
"strings"
"time"
"github.com/tevino/abool"
)
const (
repoRemoteURL = "https://gitlab.com/gitlab-org/ci-cd/tests/gitlab-test.git"
repoRefType = RefTypeBranch
repoSHA = "91956efe32fb7bef54f378d90c9bd74c19025872"
repoBeforeSHA = "ca50079dac5293292f83a4d454922ba8db44e7a3"
repoRefName = "main"
repoLFSSHA = "2371dd05e426fca09b0d2ec5d9ed757559035e2f"
repoLFSBeforeSHA = "91956efe32fb7bef54f378d90c9bd74c19025872"
repoLFSRefName = "add-lfs-object"
repoSubmoduleLFSSHA = "d0cb7ff49b5c4fcf159e860fd6b30ef40731c435"
repoSubmoduleLFSBeforeSHA = "dcbc4f0c93cb1731eeac4e3a70a55a991838e137"
repoSubmoduleLFSRefName = "add-lfs-submodule"
FilesLFSFile1LFSsize = int64(2097152)
)
var (
gitLabComChain string
gitLabComChainFetched *abool.AtomicBool
)
func init() {
gitLabComChainFetched = abool.New()
}
func GetGitInfo(url string) GitInfo {
return GitInfo{
RepoURL: url,
Sha: repoSHA,
BeforeSha: repoBeforeSHA,
Ref: repoRefName,
RefType: repoRefType,
Refspecs: []string{"+refs/heads/*:refs/origin/heads/*", "+refs/tags/*:refs/tags/*"},
}
}
func GetLFSGitInfo(url string) GitInfo {
return GitInfo{
RepoURL: url,
Sha: repoLFSSHA,
BeforeSha: repoLFSBeforeSHA,
Ref: repoLFSRefName,
RefType: repoRefType,
Refspecs: []string{"+refs/heads/*:refs/origin/heads/*", "+refs/tags/*:refs/tags/*"},
}
}
func GetSubmoduleLFSGitInfo(url string) GitInfo {
return GitInfo{
RepoURL: url,
Sha: repoSubmoduleLFSSHA,
BeforeSha: repoSubmoduleLFSBeforeSHA,
Ref: repoSubmoduleLFSRefName,
RefType: repoRefType,
Refspecs: []string{"+refs/heads/*:refs/origin/heads/*", "+refs/tags/*:refs/tags/*"},
}
}
func GetSuccessfulBuild() (JobResponse, error) {
return GetLocalBuildResponse("echo Hello World")
}
func GetRemoteSuccessfulBuild() (JobResponse, error) {
return GetRemoteBuildResponse("echo Hello World")
}
func GetRemoteSuccessfulLFSBuild() (JobResponse, error) {
response, err := GetRemoteBuildResponse("echo Hello World")
response.GitInfo = GetLFSGitInfo(repoRemoteURL)
return response, err
}
func GetRemoteSuccessfulBuildWithAfterScript() (JobResponse, error) {
jobResponse, err := GetRemoteBuildResponse("echo Hello World")
jobResponse.Steps = append(
jobResponse.Steps,
Step{
Name: StepNameAfterScript,
Script: []string{"echo Hello World"},
When: StepWhenAlways,
},
)
return jobResponse, err
}
func GetRemoteSuccessfulBuildPrintVars(shell string, vars ...string) (JobResponse, error) {
printVarsCmd := getShellPrintVars(shell, vars...)
return GetRemoteBuildResponse(printVarsCmd...)
}
func GetRemoteSuccessfulMultistepBuild() (JobResponse, error) {
jobResponse, err := GetRemoteBuildResponse("echo Hello World")
if err != nil {
return JobResponse{}, err
}
jobResponse.Steps = append(
jobResponse.Steps,
Step{
Name: "release",
Script: []string{"echo Release"},
When: StepWhenOnSuccess,
},
Step{
Name: StepNameAfterScript,
Script: []string{"echo After Script"},
When: StepWhenAlways,
},
)
return jobResponse, nil
}
func GetRemoteFailingMultistepBuild(failingStepName StepName) (JobResponse, error) {
jobResponse, err := GetRemoteSuccessfulMultistepBuild()
if err != nil {
return JobResponse{}, err
}
for i, step := range jobResponse.Steps {
if step.Name == failingStepName {
jobResponse.Steps[i].Script = append(step.Script, "exit 1")
}
}
return jobResponse, nil
}
func GetRemoteFailingMultistepBuildPrintVars(shell string, fail bool, vars ...string) (JobResponse, error) {
jobResponse, err := GetRemoteBuildResponse("echo 'Hello World'")
if err != nil {
return JobResponse{}, err
}
printVarsCmd := getShellPrintVars(shell, vars...)
exitCommand := "exit 0"
if fail {
exitCommand = "exit 1"
}
jobResponse.Steps = append(
jobResponse.Steps,
Step{
Name: "env",
Script: append(printVarsCmd, exitCommand),
When: StepWhenOnSuccess,
},
Step{
Name: StepNameAfterScript,
Script: printVarsCmd,
When: StepWhenAlways,
},
)
return jobResponse, nil
}
func getShellPrintVars(shell string, vars ...string) []string {
var envCommand []string
var fmtStr string
switch shell {
case "cmd":
// Using double %% so % is escaped in fmt.
fmtStr = "echo %s=%%%s%%"
case "powershell", "pwsh":
fmtStr = "echo %s=$env:%s"
default:
fmtStr = "echo %s=$%s"
}
for _, v := range vars {
envCommand = append(envCommand, fmt.Sprintf(fmtStr, v, v))
}
return envCommand
}
func GetRemoteSuccessfulBuildWithDumpedVariables() (JobResponse, error) {
variableName := "test_dump"
variableValue := "test"
response, err := GetRemoteBuildResponse(
fmt.Sprintf("[[ \"${%s}\" != \"\" ]]", variableName),
fmt.Sprintf("[[ $(cat $%s) == \"%s\" ]]", variableName, variableValue),
)
if err != nil {
return JobResponse{}, err
}
dumpedVariable := JobVariable{
Key: variableName, Value: variableValue,
Internal: true, Public: true, File: true,
}
response.Variables = append(response.Variables, dumpedVariable)
return response, nil
}
func GetFailedBuild() (JobResponse, error) {
return GetLocalBuildResponse("exit 1")
}
func GetRemoteFailedBuild() (JobResponse, error) {
return GetRemoteBuildResponse("exit 1")
}
func GetLongRunningBuild() (JobResponse, error) {
return GetLocalBuildResponse("sleep 3600")
}
func GetRemoteLongRunningBuild() (JobResponse, error) {
return GetRemoteBuildResponse("sleep 3600")
}
func GetRemoteLongRunningBuildCMD() (JobResponse, error) {
// Can't use TIMEOUT since it requires input redirection,
// https://knowledge.broadcom.com/external/article/29524/the-timeout-command-in-batch-script-job.html
return GetLocalBuildResponse("ping 127.0.0.1 -n 3600 > nul")
}
func GetRemoteLongRunningBuildWithAfterScript() (JobResponse, error) {
jobResponse, err := GetRemoteLongRunningBuild()
if err != nil {
return JobResponse{}, err
}
addAfterScript(&jobResponse)
return jobResponse, nil
}
func GetRemoteLongRunningBuildWithAfterScriptCMD() (JobResponse, error) {
jobResponse, err := GetRemoteLongRunningBuildCMD()
if err != nil {
return JobResponse{}, err
}
addAfterScript(&jobResponse)
return jobResponse, nil
}
func addAfterScript(jobResponse *JobResponse) {
jobResponse.Steps = append(
jobResponse.Steps,
Step{
Name: StepNameAfterScript,
Script: []string{
"echo Hello World from after_script",
},
When: StepWhenAlways,
},
)
}
func GetMultilineBashBuild() (JobResponse, error) {
return GetRemoteBuildResponse(`if true; then
echo 'Hello World'
fi
`)
}
func GetMultilineBashBuildPowerShell() (JobResponse, error) {
return GetRemoteBuildResponse("if (0 -eq 0) {\n\recho \"Hello World\"\n\r}")
}
func GetMultilineBashBuildCmd() (JobResponse, error) {
return GetRemoteBuildResponse(`IF 0==0 (
echo Hello World
)`)
}
func GetRemoteBrokenTLSBuild() (JobResponse, error) {
invalidCert, err := buildSnakeOilCert()
if err != nil {
return JobResponse{}, err
}
return getRemoteCustomTLSBuild(invalidCert)
}
func GetRemoteGitLabComTLSBuild() (JobResponse, error) {
cert, err := getGitLabComTLSChain()
if err != nil {
return JobResponse{}, err
}
return getRemoteCustomTLSBuild(cert)
}
func getRemoteCustomTLSBuild(chain string) (JobResponse, error) {
job, err := GetRemoteBuildResponse("echo Hello World")
if err != nil {
return JobResponse{}, err
}
job.TLSCAChain = chain
job.Variables = append(
job.Variables,
JobVariable{Key: "GIT_STRATEGY", Value: "clone"},
JobVariable{Key: "GIT_SUBMODULE_STRATEGY", Value: "normal"},
)
return job, nil
}
func getBuildResponse(repoURL string, commands []string) JobResponse {
return JobResponse{
GitInfo: GetGitInfo(repoURL),
Steps: Steps{
Step{
Name: StepNameScript,
Script: commands,
When: StepWhenAlways,
AllowFailure: false,
},
},
}
}
func GetRemoteBuildResponse(commands ...string) (JobResponse, error) {
return getBuildResponse(repoRemoteURL, commands), nil
}
func GetLocalBuildResponse(commands ...string) (JobResponse, error) {
localRepoURL, err := getLocalRepoURL()
if err != nil {
if os.IsNotExist(err) {
panic("Local repo not found, please run `make development_setup`")
}
return JobResponse{}, err
}
return getBuildResponse(localRepoURL, commands), nil
}
func getLocalRepoURL() (string, error) {
_, filename, _, _ := runtime.Caller(0) //nolint:dogsled
directory := path.Dir(filename)
if strings.Contains(directory, "_test/_obj_test") {
pwd, err := os.Getwd()
if err != nil {
return "", err
}
directory = pwd
}
localRepoURL := path.Clean(directory + "/../tmp/gitlab-test/.git")
_, err := os.Stat(localRepoURL)
if err != nil {
return "", err
}
return localRepoURL, nil
}
func buildSnakeOilCert() (string, error) {
priv, err := rsa.GenerateKey(rand.Reader, 1024)
if err != nil {
return "", err
}
notBefore := time.Now()
notAfter := notBefore.Add(time.Hour)
template := x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
Organization: []string{"Snake Oil Co"},
},
NotBefore: notBefore,
NotAfter: notAfter,
IsCA: true,
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
}
derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
if err != nil {
return "", err
}
certificate := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
return string(certificate), nil
}
func getGitLabComTLSChain() (string, error) {
if gitLabComChainFetched.IsSet() {
return gitLabComChain, nil
}
resp, err := http.Head("https://gitlab.com/users/sign_in")
if err != nil {
return "", err
}
defer func() { _ = resp.Body.Close() }()
var buff bytes.Buffer
for _, certs := range resp.TLS.VerifiedChains {
for _, cert := range certs {
err = pem.Encode(&buff, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})
if err != nil {
return "", err
}
}
}
gitLabComChain = buff.String()
gitLabComChainFetched.Set()
return gitLabComChain, nil
}
package common
import (
"context"
"io"
"os"
"sync"
)
type Trace struct {
Writer io.Writer
cancelFunc context.CancelFunc
abortFunc context.CancelFunc
mutex sync.Mutex
}
type masker interface {
SetMasked([]string)
}
type JobFailureData struct {
Reason JobFailureReason
ExitCode int
}
func (s *Trace) Write(p []byte) (n int, err error) {
s.mutex.Lock()
defer s.mutex.Unlock()
if s.Writer == nil {
return 0, os.ErrInvalid
}
return s.Writer.Write(p)
}
func (s *Trace) SetMasked(values []string) {
masker, ok := s.Writer.(masker)
if ok {
masker.SetMasked(values)
}
}
func (s *Trace) Success() {
}
func (s *Trace) Fail(err error, failureData JobFailureData) {
}
func (s *Trace) SetCancelFunc(cancelFunc context.CancelFunc) {
s.mutex.Lock()
defer s.mutex.Unlock()
s.cancelFunc = cancelFunc
}
func (s *Trace) Cancel() bool {
s.mutex.Lock()
defer s.mutex.Unlock()
if s.cancelFunc == nil {
return false
}
s.cancelFunc()
return true
}
func (s *Trace) SetAbortFunc(abortFunc context.CancelFunc) {
s.mutex.Lock()
defer s.mutex.Unlock()
s.abortFunc = abortFunc
}
func (s *Trace) Abort() bool {
s.mutex.Lock()
defer s.mutex.Unlock()
if s.abortFunc == nil {
return false
}
// Abort always have much higher importance than Cancel
// as abort interrupts the execution
s.cancelFunc = nil
s.abortFunc()
return true
}
func (s *Trace) SetFailuresCollector(fc FailuresCollector) {}
func (s *Trace) IsStdout() bool {
return true
}
package common
import (
"errors"
"fmt"
"os"
"strings"
)
type JobVariable struct {
Key string `json:"key"`
Value string `json:"value"`
Public bool `json:"public"`
Internal bool `json:"-"`
File bool `json:"file"`
Masked bool `json:"masked"`
Raw bool `json:"raw"`
}
type JobVariables []JobVariable
func (b JobVariable) String() string {
return fmt.Sprintf("%s=%s", b.Key, b.Value)
}
func (b JobVariables) PublicOrInternal() (variables JobVariables) {
for _, variable := range b {
if variable.Public || variable.Internal {
variables = append(variables, variable)
}
}
return variables
}
func (b JobVariables) StringList() (variables []string) {
for _, variable := range b {
variables = append(variables, variable.String())
}
return variables
}
func (b JobVariables) Get(key string) string {
switch key {
case "$":
return key
case "*", "#", "@", "!", "?", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9":
return ""
}
for i := len(b) - 1; i >= 0; i-- {
if b[i].Key == key {
return b[i].Value
}
}
return ""
}
// OverwriteKey overwrites an existing key with a new variable.
func (b JobVariables) OverwriteKey(key string, variable JobVariable) {
for i, v := range b {
if v.Key == key {
b[i] = variable
return
}
}
}
func (b JobVariables) ExpandValue(value string) string {
return os.Expand(value, b.Get)
}
func (b JobVariables) Expand() JobVariables {
var variables JobVariables
for _, variable := range b {
if !variable.Raw {
variable.Value = b.ExpandValue(variable.Value)
}
variables = append(variables, variable)
}
return variables
}
func (b JobVariables) Masked() (masked []string) {
for _, variable := range b {
if variable.Masked {
masked = append(masked, variable.Value)
}
}
return
}
func ParseVariable(text string) (variable JobVariable, err error) {
keyValue := strings.SplitN(text, "=", 2)
if len(keyValue) != 2 {
err = errors.New("missing =")
return
}
variable = JobVariable{
Key: keyValue[0],
Value: keyValue[1],
}
return
}
package common
import (
"fmt"
"runtime"
"github.com/prometheus/client_golang/prometheus"
"github.com/urfave/cli"
)
var NAME = "gitlab-runner"
var VERSION = "development version"
var REVISION = "HEAD"
var BRANCH = "HEAD"
var BUILT = "unknown"
var AppVersion AppVersionInfo
type AppVersionInfo struct {
Name string `json:"name"`
Version string `json:"version"`
Revision string `json:"revision"`
Branch string `json:"branch"`
GOVersion string `json:"go_version"`
BuiltAt string `json:"built_at"`
OS string `json:"os"`
Architecture string `json:"architecture"`
}
func (v *AppVersionInfo) Printer(c *cli.Context) {
fmt.Print(v.Extended())
}
func (v *AppVersionInfo) Line() string {
return fmt.Sprintf("%s %s (%s)", v.Name, v.Version, v.Revision)
}
func (v *AppVersionInfo) ShortLine() string {
return fmt.Sprintf("%s (%s)", v.Version, v.Revision)
}
func (v *AppVersionInfo) UserAgent() string {
return fmt.Sprintf("%s %s (%s; %s; %s/%s)", v.Name, v.Version, v.Branch, v.GOVersion, v.OS, v.Architecture)
}
func (v *AppVersionInfo) Variables() JobVariables {
return JobVariables{
{Key: "CI_RUNNER_VERSION", Value: v.Version, Public: true, Internal: true, File: false},
{Key: "CI_RUNNER_REVISION", Value: v.Revision, Public: true, Internal: true, File: false},
{
Key: "CI_RUNNER_EXECUTABLE_ARCH",
Value: fmt.Sprintf("%s/%s", v.OS, v.Architecture),
Public: true,
Internal: true,
File: false,
},
}
}
func (v *AppVersionInfo) Extended() string {
version := fmt.Sprintf("Version: %s\n", v.Version)
version += fmt.Sprintf("Git revision: %s\n", v.Revision)
version += fmt.Sprintf("Git branch: %s\n", v.Branch)
version += fmt.Sprintf("GO version: %s\n", v.GOVersion)
version += fmt.Sprintf("Built: %s\n", v.BuiltAt)
version += fmt.Sprintf("OS/Arch: %s/%s\n", v.OS, v.Architecture)
return version
}
// NewMetricsCollector returns a prometheus.Collector which represents current build information.
func (v *AppVersionInfo) NewMetricsCollector() *prometheus.GaugeVec {
labels := map[string]string{
"name": v.Name,
"version": v.Version,
"revision": v.Revision,
"branch": v.Branch,
"go_version": v.GOVersion,
"built_at": v.BuiltAt,
"os": v.OS,
"architecture": v.Architecture,
}
labelNames := make([]string, 0, len(labels))
for n := range labels {
labelNames = append(labelNames, n)
}
buildInfo := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "gitlab_runner_version_info",
Help: "A metric with a constant '1' value labeled by different build stats fields.",
},
labelNames,
)
buildInfo.With(labels).Set(1)
return buildInfo
}
func init() {
AppVersion = AppVersionInfo{
Name: NAME,
Version: VERSION,
Revision: REVISION,
Branch: BRANCH,
GOVersion: runtime.Version(),
BuiltAt: BUILT,
OS: runtime.GOOS,
Architecture: runtime.GOARCH,
}
}
package executors
import (
"context"
"os"
"sync"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/session/proxy"
)
type ExecutorOptions struct {
DefaultCustomBuildsDirEnabled bool
DefaultBuildsDir string
DefaultCacheDir string
SharedBuildsDir bool
Shell common.ShellScriptInfo
ShowHostname bool
Metadata map[string]string
}
type AbstractExecutor struct {
ExecutorOptions
common.BuildLogger
Config common.RunnerConfig
Build *common.Build
Trace common.JobTrace
BuildShell *common.ShellConfiguration
currentStage common.ExecutorStage
Context context.Context
ProxyPool proxy.Pool
stageLock sync.RWMutex
}
func (e *AbstractExecutor) updateShell() error {
script := e.Shell()
script.Build = e.Build
if e.Config.Shell != "" {
script.Shell = e.Config.Shell
}
return nil
}
func (e *AbstractExecutor) generateShellConfiguration() error {
info := e.Shell()
info.PreCloneScript = e.Config.PreCloneScript
info.PreBuildScript = e.Config.PreBuildScript
info.PostBuildScript = e.Config.PostBuildScript
shellConfiguration, err := common.GetShellConfiguration(*info)
if err != nil {
return err
}
e.BuildShell = shellConfiguration
e.Debugln("Shell configuration:", shellConfiguration)
return nil
}
func (e *AbstractExecutor) startBuild() error {
// Save hostname
if e.ShowHostname && e.Build.Hostname == "" {
e.Build.Hostname, _ = os.Hostname()
}
return e.Build.StartBuild(
e.RootDir(),
e.CacheDir(),
e.CustomBuildEnabled(),
e.SharedBuildsDir,
)
}
func (e *AbstractExecutor) RootDir() string {
if e.Config.BuildsDir != "" {
return e.Config.BuildsDir
}
return e.DefaultBuildsDir
}
func (e *AbstractExecutor) CacheDir() string {
if e.Config.CacheDir != "" {
return e.Config.CacheDir
}
return e.DefaultCacheDir
}
func (e *AbstractExecutor) CustomBuildEnabled() bool {
if e.Config.CustomBuildDir != nil {
return e.Config.CustomBuildDir.Enabled
}
return e.DefaultCustomBuildsDirEnabled
}
func (e *AbstractExecutor) Shell() *common.ShellScriptInfo {
return &e.ExecutorOptions.Shell
}
func (e *AbstractExecutor) Prepare(options common.ExecutorPrepareOptions) error {
e.PrepareConfiguration(options)
return e.PrepareBuildAndShell()
}
func (e *AbstractExecutor) PrepareConfiguration(options common.ExecutorPrepareOptions) {
e.SetCurrentStage(common.ExecutorStagePrepare)
e.Context = options.Context
e.Config = *options.Config
e.Build = options.Build
e.Trace = options.Trace
e.BuildLogger = common.NewBuildLogger(options.Trace, options.Build.Log())
e.ProxyPool = proxy.NewPool()
}
func (e *AbstractExecutor) PrepareBuildAndShell() error {
err := e.startBuild()
if err != nil {
return err
}
err = e.updateShell()
if err != nil {
return err
}
err = e.generateShellConfiguration()
if err != nil {
return err
}
return nil
}
func (e *AbstractExecutor) Finish(err error) {
e.SetCurrentStage(common.ExecutorStageFinish)
}
func (e *AbstractExecutor) Cleanup() {
e.SetCurrentStage(common.ExecutorStageCleanup)
}
func (e *AbstractExecutor) GetCurrentStage() common.ExecutorStage {
e.stageLock.RLock()
defer e.stageLock.RUnlock()
return e.currentStage
}
func (e *AbstractExecutor) SetCurrentStage(stage common.ExecutorStage) {
e.stageLock.Lock()
defer e.stageLock.Unlock()
e.currentStage = stage
}
package command
import (
"context"
"fmt"
"os"
"os/exec"
"strconv"
"time"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors/custom/api"
"gitlab.com/gitlab-org/gitlab-runner/helpers/process"
)
const (
BuildFailureExitCode = 1
SystemFailureExitCode = 2
)
type Command interface {
Run() error
}
var newProcessKillWaiter = process.NewOSKillWait
var newCommander = process.NewOSCmd
type Options struct {
JobResponseFile string
}
type command struct {
context context.Context
cmd process.Commander
waitCh chan error
logger process.Logger
gracefulKillTimeout time.Duration
forceKillTimeout time.Duration
}
func New(
ctx context.Context,
executable string,
args []string,
cmdOpts process.CommandOptions,
options Options,
) Command {
defaultVariables := map[string]string{
"TMPDIR": cmdOpts.Dir,
api.BuildFailureExitCodeVariable: strconv.Itoa(BuildFailureExitCode),
api.SystemFailureExitCodeVariable: strconv.Itoa(SystemFailureExitCode),
api.JobResponseFileVariable: options.JobResponseFile,
}
env := os.Environ()
for key, value := range defaultVariables {
env = append(env, fmt.Sprintf("%s=%s", key, value))
}
cmdOpts.Env = append(env, cmdOpts.Env...)
return &command{
context: ctx,
cmd: newCommander(executable, args, cmdOpts),
waitCh: make(chan error),
logger: cmdOpts.Logger,
gracefulKillTimeout: cmdOpts.GracefulKillTimeout,
forceKillTimeout: cmdOpts.ForceKillTimeout,
}
}
func (c *command) Run() error {
err := c.cmd.Start()
if err != nil {
return fmt.Errorf("failed to start command: %w", err)
}
go c.waitForCommand()
select {
case err = <-c.waitCh:
return err
case <-c.context.Done():
return newProcessKillWaiter(c.logger, c.gracefulKillTimeout, c.forceKillTimeout).
KillAndWait(c.cmd, c.waitCh)
}
}
var getExitCode = func(err *exec.ExitError) int {
return err.ExitCode()
}
func (c *command) waitForCommand() {
err := c.cmd.Wait()
eerr, ok := err.(*exec.ExitError)
if ok {
exitCode := getExitCode(eerr)
switch {
case exitCode == BuildFailureExitCode:
err = &common.BuildError{Inner: eerr, ExitCode: exitCode}
case exitCode != SystemFailureExitCode:
err = &ErrUnknownFailure{Inner: eerr, ExitCode: exitCode}
}
}
c.waitCh <- err
}
package command
import (
"fmt"
)
type ErrUnknownFailure struct {
Inner error
ExitCode int
}
func (e *ErrUnknownFailure) Error() string {
return fmt.Sprintf(
"unknown Custom executor executable exit code %d; executable execution terminated with: %v",
e.ExitCode,
e.Inner,
)
}
package custom
import (
"time"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/process"
)
type config struct {
*common.CustomConfig
}
func (c *config) GetConfigExecTimeout() time.Duration {
return getDuration(c.ConfigExecTimeout, defaultConfigExecTimeout)
}
func (c *config) GetPrepareExecTimeout() time.Duration {
return getDuration(c.PrepareExecTimeout, defaultPrepareExecTimeout)
}
func (c *config) GetCleanupScriptTimeout() time.Duration {
return getDuration(c.CleanupExecTimeout, defaultCleanupExecTimeout)
}
func (c *config) GetGracefulKillTimeout() time.Duration {
return getDuration(c.GracefulKillTimeout, process.GracefulTimeout)
}
func (c *config) GetForceKillTimeout() time.Duration {
return getDuration(c.ForceKillTimeout, process.KillTimeout)
}
func getDuration(source *int, defaultValue time.Duration) time.Duration {
if source == nil {
return defaultValue
}
timeout := *source
if timeout <= 0 {
return defaultValue
}
return time.Duration(timeout) * time.Second
}
package custom
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors"
"gitlab.com/gitlab-org/gitlab-runner/executors/custom/api"
"gitlab.com/gitlab-org/gitlab-runner/executors/custom/command"
"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
"gitlab.com/gitlab-org/gitlab-runner/helpers/process"
)
type commandOutputs struct {
stdout io.Writer
stderr io.Writer
}
type prepareCommandOpts struct {
executable string
args []string
out commandOutputs
}
type ConfigExecOutput struct {
api.ConfigExecOutput
}
type jsonService struct {
Name string `json:"name"`
Alias string `json:"alias"`
Entrypoint []string `json:"entrypoint"`
Command []string `json:"command"`
}
func (c *ConfigExecOutput) InjectInto(executor *executor) {
if c.Hostname != nil {
executor.Build.Hostname = *c.Hostname
}
if c.BuildsDir != nil {
executor.Config.BuildsDir = *c.BuildsDir
}
if c.CacheDir != nil {
executor.Config.CacheDir = *c.CacheDir
}
if c.BuildsDirIsShared != nil {
executor.SharedBuildsDir = *c.BuildsDirIsShared
}
executor.driverInfo = c.Driver
if c.JobEnv != nil {
executor.jobEnv = *c.JobEnv
}
}
type executor struct {
executors.AbstractExecutor
config *config
tempDir string
jobResponseFile string
driverInfo *api.DriverInfo
jobEnv map[string]string
}
func (e *executor) Prepare(options common.ExecutorPrepareOptions) error {
e.AbstractExecutor.PrepareConfiguration(options)
err := e.prepareConfig()
if err != nil {
return err
}
e.tempDir, err = ioutil.TempDir("", "custom-executor")
if err != nil {
return err
}
e.jobResponseFile, err = e.createJobResponseFile()
if err != nil {
return err
}
err = e.dynamicConfig()
if err != nil {
return err
}
e.logStartupMessage()
err = e.AbstractExecutor.PrepareBuildAndShell()
if err != nil {
return err
}
// nothing to do, as there's no prepare_script
if e.config.PrepareExec == "" {
return nil
}
ctx, cancelFunc := context.WithTimeout(e.Context, e.config.GetPrepareExecTimeout())
defer cancelFunc()
opts := prepareCommandOpts{
executable: e.config.PrepareExec,
args: e.config.PrepareArgs,
out: e.defaultCommandOutputs(),
}
return e.prepareCommand(ctx, opts).Run()
}
func (e *executor) prepareConfig() error {
if e.Config.Custom == nil {
return common.MakeBuildError("custom executor not configured")
}
e.config = &config{
CustomConfig: e.Config.Custom,
}
if e.config.RunExec == "" {
return common.MakeBuildError("custom executor is missing RunExec")
}
return nil
}
func (e *executor) createJobResponseFile() (string, error) {
responseFile := filepath.Join(e.tempDir, "response.json")
file, err := os.OpenFile(responseFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return "", fmt.Errorf("creating job response file %q: %w", responseFile, err)
}
defer func() { _ = file.Close() }()
encoder := json.NewEncoder(file)
err = encoder.Encode(e.Build.JobResponse)
if err != nil {
return "", fmt.Errorf("encoding job response file: %w", err)
}
return responseFile, nil
}
func (e *executor) dynamicConfig() error {
if e.config.ConfigExec == "" {
return nil
}
ctx, cancelFunc := context.WithTimeout(e.Context, e.config.GetConfigExecTimeout())
defer cancelFunc()
buf := bytes.NewBuffer(nil)
outputs := commandOutputs{
stdout: buf,
stderr: e.Trace,
}
opts := prepareCommandOpts{
executable: e.config.ConfigExec,
args: e.config.ConfigArgs,
out: outputs,
}
err := e.prepareCommand(ctx, opts).Run()
if err != nil {
return err
}
jsonConfig := buf.Bytes()
if len(jsonConfig) < 1 {
return nil
}
config := new(ConfigExecOutput)
err = json.Unmarshal(jsonConfig, config)
if err != nil {
return fmt.Errorf("error while parsing JSON output: %w", err)
}
config.InjectInto(e)
return nil
}
func (e *executor) logStartupMessage() {
const usageLine = "Using Custom executor"
info := e.driverInfo
if info == nil || info.Name == nil {
e.Println(fmt.Sprintf("%s...", usageLine))
return
}
if info.Version == nil {
e.Println(fmt.Sprintf("%s with driver %s...", usageLine, *info.Name))
return
}
e.Println(fmt.Sprintf("%s with driver %s %s...", usageLine, *info.Name, *info.Version))
}
func (e *executor) defaultCommandOutputs() commandOutputs {
return commandOutputs{
stdout: e.Trace,
stderr: e.Trace,
}
}
var commandFactory = command.New
func (e *executor) prepareCommand(ctx context.Context, opts prepareCommandOpts) command.Command {
logger := common.NewProcessLoggerAdapter(e.BuildLogger)
cmdOpts := process.CommandOptions{
Dir: e.tempDir,
Env: make([]string, 0),
Stdout: opts.out.stdout,
Stderr: opts.out.stderr,
Logger: logger,
GracefulKillTimeout: e.config.GetGracefulKillTimeout(),
ForceKillTimeout: e.config.GetForceKillTimeout(),
UseWindowsLegacyProcessStrategy: e.Build.IsFeatureFlagOn(featureflags.UseWindowsLegacyProcessStrategy),
}
// Append job_env defined variable first to avoid overwriting any CI/CD or predefined variables.
for k, v := range e.jobEnv {
cmdOpts.Env = append(cmdOpts.Env, fmt.Sprintf("%s=%s", k, v))
}
variables := append(e.Build.GetAllVariables(), e.getCIJobServicesEnv())
for _, variable := range variables {
cmdOpts.Env = append(cmdOpts.Env, fmt.Sprintf("CUSTOM_ENV_%s=%s", variable.Key, variable.Value))
}
options := command.Options{
JobResponseFile: e.jobResponseFile,
}
return commandFactory(ctx, opts.executable, opts.args, cmdOpts, options)
}
func (e *executor) getCIJobServicesEnv() common.JobVariable {
if len(e.Build.Services) == 0 {
return common.JobVariable{Key: "CI_JOB_SERVICES"}
}
var services []jsonService
for _, service := range e.Build.Services {
services = append(services, jsonService{
Name: service.Name,
Alias: service.Alias,
Entrypoint: service.Entrypoint,
Command: service.Command,
})
}
servicesSerialized, err := json.Marshal(services)
if err != nil {
e.Warningln("Unable to serialize CI_JOB_SERVICES json:", err)
}
return common.JobVariable{
Key: "CI_JOB_SERVICES",
Value: string(servicesSerialized),
}
}
func (e *executor) Run(cmd common.ExecutorCommand) error {
scriptDir, err := ioutil.TempDir(e.tempDir, "script")
if err != nil {
return err
}
scriptFile := filepath.Join(scriptDir, "script."+e.BuildShell.Extension)
err = ioutil.WriteFile(scriptFile, []byte(cmd.Script), 0700)
if err != nil {
return err
}
// TODO: Remove this translation in 14.0 - https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26426
stage := cmd.Stage
if stage == "step_script" {
e.BuildLogger.Warningln("Starting with version 14.0 the 'build_script' stage " +
"will be replaced with 'step_script': https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26426")
stage = "build_script"
}
args := append(e.config.RunArgs, scriptFile, string(stage))
opts := prepareCommandOpts{
executable: e.config.RunExec,
args: args,
out: e.defaultCommandOutputs(),
}
return e.prepareCommand(cmd.Context, opts).Run()
}
func (e *executor) Cleanup() {
e.AbstractExecutor.Cleanup()
err := e.prepareConfig()
if err != nil {
e.Warningln(err)
// at this moment we don't care about the errors
return
}
defer func() { _ = os.RemoveAll(e.tempDir) }()
// nothing to do, as there's no cleanup_script
if e.config.CleanupExec == "" {
return
}
ctx, cancelFunc := context.WithTimeout(context.Background(), e.config.GetCleanupScriptTimeout())
defer cancelFunc()
stdoutLogger := e.BuildLogger.WithFields(logrus.Fields{"cleanup_std": "out"})
stderrLogger := e.BuildLogger.WithFields(logrus.Fields{"cleanup_std": "err"})
outputs := commandOutputs{
stdout: stdoutLogger.WriterLevel(logrus.DebugLevel),
stderr: stderrLogger.WriterLevel(logrus.WarnLevel),
}
opts := prepareCommandOpts{
executable: e.config.CleanupExec,
args: e.config.CleanupArgs,
out: outputs,
}
err = e.prepareCommand(ctx, opts).Run()
if err != nil {
e.Warningln("Cleanup script failed:", err)
}
}
func init() {
options := executors.ExecutorOptions{
DefaultCustomBuildsDirEnabled: false,
Shell: common.ShellScriptInfo{
Shell: common.GetDefaultShell(),
Type: common.NormalShell,
RunnerCommand: "gitlab-runner",
},
ShowHostname: false,
}
creator := func() common.Executor {
return &executor{
AbstractExecutor: executors.AbstractExecutor{
ExecutorOptions: options,
},
}
}
featuresUpdater := func(features *common.FeaturesInfo) {
features.Variables = true
features.Shared = true
}
common.RegisterExecutorProvider("custom", executors.DefaultExecutorProvider{
Creator: creator,
FeaturesUpdater: featuresUpdater,
DefaultShellName: options.Shell.Shell,
})
}
// +build !windows
package custom
import (
"errors"
terminalsession "gitlab.com/gitlab-org/gitlab-runner/session/terminal"
)
func (e *executor) Connect() (terminalsession.Conn, error) {
return nil, errors.New("not yet supported")
}
package executors
import (
"errors"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type DefaultExecutorProvider struct {
Creator func() common.Executor
FeaturesUpdater func(features *common.FeaturesInfo)
ConfigUpdater func(input *common.RunnerConfig, output *common.ConfigInfo)
DefaultShellName string
}
func (e DefaultExecutorProvider) CanCreate() bool {
return e.Creator != nil
}
func (e DefaultExecutorProvider) Create() common.Executor {
if e.Creator == nil {
return nil
}
return e.Creator()
}
func (e DefaultExecutorProvider) Acquire(config *common.RunnerConfig) (common.ExecutorData, error) {
return nil, nil
}
func (e DefaultExecutorProvider) Release(config *common.RunnerConfig, data common.ExecutorData) {}
func (e DefaultExecutorProvider) GetFeatures(features *common.FeaturesInfo) error {
if e.FeaturesUpdater == nil {
return errors.New("cannot evaluate features")
}
e.FeaturesUpdater(features)
return nil
}
func (e DefaultExecutorProvider) GetConfigInfo(input *common.RunnerConfig, output *common.ConfigInfo) {
if e.ConfigUpdater == nil {
return
}
e.ConfigUpdater(input, output)
}
func (e DefaultExecutorProvider) GetDefaultShell() string {
return e.DefaultShellName
}
package docker
import (
"strings"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
func configUpdater(input *common.RunnerConfig, output *common.ConfigInfo) {
if input.RunnerSettings.Docker != nil {
output.Gpus = strings.Trim(input.RunnerSettings.Docker.Gpus, " ")
}
}
package docker
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/docker/cli/opts"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/pkg/stdcopy"
"github.com/kardianos/osext"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/exec"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/labels"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/networks"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/pull"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/permission"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/wait"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
"gitlab.com/gitlab-org/gitlab-runner/helpers/container/helperimage"
"gitlab.com/gitlab-org/gitlab-runner/helpers/container/services"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
"gitlab.com/gitlab-org/gitlab-runner/shells"
)
const (
ExecutorStagePrepare common.ExecutorStage = "docker_prepare"
ExecutorStageRun common.ExecutorStage = "docker_run"
ExecutorStageCleanup common.ExecutorStage = "docker_cleanup"
ExecutorStageCreatingBuildVolumes common.ExecutorStage = "docker_creating_build_volumes"
ExecutorStageCreatingServices common.ExecutorStage = "docker_creating_services"
ExecutorStageCreatingUserVolumes common.ExecutorStage = "docker_creating_user_volumes"
ExecutorStagePullingImage common.ExecutorStage = "docker_pulling_image"
)
var PrebuiltImagesPaths []string
const (
labelServiceType = "service"
labelWaitType = "wait"
)
var neverRestartPolicy = container.RestartPolicy{Name: "no"}
var (
errVolumesManagerUndefined = errors.New("volumesManager is undefined")
errNetworksManagerUndefined = errors.New("networksManager is undefined")
)
type executor struct {
executors.AbstractExecutor
client docker.Client
volumeParser parser.Parser
newVolumePermissionSetter func() (permission.Setter, error)
info types.Info
waiter wait.KillWaiter
temporary []string // IDs of containers that should be removed
builds []string // IDs of successfully created build containers
services []*types.Container
links []string
devices []container.DeviceMapping
deviceRequests []container.DeviceRequest
helperImageInfo helperimage.Info
volumesManager volumes.Manager
networksManager networks.Manager
labeler labels.Labeler
pullManager pull.Manager
networkMode container.NetworkMode
projectUniqRandomizedName string
}
func init() {
runnerFolder, err := osext.ExecutableFolder()
if err != nil {
logrus.Errorln(
"Docker executor: unable to detect gitlab-runner folder, "+
"prebuilt image helpers will be loaded from remote registry.",
err,
)
}
PrebuiltImagesPaths = []string{
// When gitlab-runner is running from repository root
filepath.Join(runnerFolder, "out/helper-images"),
// When gitlab-runner is running from `out/binaries`
filepath.Join(runnerFolder, "../helper-images"),
// Add working directory path, used when running from temp directory, such as with `go run`
filepath.Join(helpers.GetCurrentWorkingDirectory(), "out/helper-images"),
}
if runtime.GOOS == "linux" {
// This section covers the Linux packaged app scenario, with the binary in /usr/bin.
// The helper images are located in /usr/lib/gitlab-runner/helper-images,
// as part of the packaging done in the create_package function in ci/package
PrebuiltImagesPaths = append(
PrebuiltImagesPaths,
filepath.Join(runnerFolder, "../lib/gitlab-runner/helper-images"),
)
}
}
func (e *executor) getServiceVariables() []string {
return e.Build.GetAllVariables().PublicOrInternal().StringList()
}
func (e *executor) expandAndGetDockerImage(imageName string, allowedImages []string) (*types.ImageInspect, error) {
imageName, err := e.expandImageName(imageName, allowedImages)
if err != nil {
return nil, err
}
image, err := e.pullManager.GetDockerImage(imageName)
if err != nil {
return nil, err
}
return image, nil
}
func (e *executor) loadPrebuiltImage(path, ref, tag string) (*types.ImageInspect, error) {
file, err := os.OpenFile(path, os.O_RDONLY, 0600)
if err != nil {
if os.IsNotExist(err) {
return nil, err
}
return nil, fmt.Errorf("cannot load prebuilt image: %s: %w", path, err)
}
defer func() { _ = file.Close() }()
e.Debugln("Loading prebuilt image...")
source := types.ImageImportSource{
Source: file,
SourceName: "-",
}
options := types.ImageImportOptions{
Tag: tag,
// NOTE: The ENTRYPOINT metadata is not preserved on export, so we need to reapply this metadata on import.
// See https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/2058#note_388341301
Changes: []string{`ENTRYPOINT ["/usr/bin/dumb-init", "/entrypoint"]`},
}
if err = e.client.ImageImportBlocking(e.Context, source, ref, options); err != nil {
return nil, fmt.Errorf("failed to import image: %w", err)
}
image, _, err := e.client.ImageInspectWithRaw(e.Context, ref+":"+tag)
if err != nil {
e.Debugln("Inspecting imported image", ref, "failed:", err)
return nil, err
}
return &image, err
}
func (e *executor) getPrebuiltImage() (*types.ImageInspect, error) {
if imageNameFromConfig := e.Config.Docker.HelperImage; imageNameFromConfig != "" {
imageNameFromConfig = common.AppVersion.Variables().ExpandValue(imageNameFromConfig)
e.Debugln(
"Pull configured helper_image for predefined container instead of import bundled image",
imageNameFromConfig,
"...",
)
return e.pullManager.GetDockerImage(imageNameFromConfig)
}
e.Debugln(fmt.Sprintf("Looking for prebuilt image %s...", e.helperImageInfo))
image, _, err := e.client.ImageInspectWithRaw(e.Context, e.helperImageInfo.String())
if err == nil {
return &image, nil
}
// Try to load prebuilt image from local filesystem
loadedImage := e.getLocalHelperImage()
if loadedImage != nil {
return loadedImage, nil
}
if !e.Build.IsFeatureFlagOn(featureflags.GitLabRegistryHelperImage) {
e.Warningln(helperimage.DockerHubWarningMessage)
}
// Fall back to getting image from registry
e.Debugln(fmt.Sprintf("Loading image form registry: %s", e.helperImageInfo))
return e.pullManager.GetDockerImage(e.helperImageInfo.String())
}
func (e *executor) getLocalHelperImage() *types.ImageInspect {
if !e.helperImageInfo.IsSupportingLocalImport {
return nil
}
var flavor string
if e.Config.Docker != nil {
flavor = e.Config.Docker.HelperImageFlavor
}
architecture := e.helperImageInfo.Architecture
prebuiltFileName := getPrebuiltFileName(architecture, flavor, e.Config.Shell)
for _, dockerPrebuiltImagesPath := range PrebuiltImagesPaths {
dockerPrebuiltImageFilePath := filepath.Join(dockerPrebuiltImagesPath, prebuiltFileName)
image, err := e.loadPrebuiltImage(
dockerPrebuiltImageFilePath,
e.helperImageInfo.Name,
e.helperImageInfo.Tag,
)
if err != nil {
e.Debugln("Failed to load prebuilt image from:", dockerPrebuiltImageFilePath, "error:", err)
continue
}
return image
}
return nil
}
func getPrebuiltFileName(architecture, flavor string, shell string) string {
if flavor == "" {
flavor = helperimage.DefaultFlavor
}
if shell == shells.SNPwsh {
return fmt.Sprintf("prebuilt-%s-%s-%s%s", flavor, architecture, shell, prebuiltImageExtension)
}
return fmt.Sprintf("prebuilt-%s-%s%s", flavor, architecture, prebuiltImageExtension)
}
func (e *executor) getBuildImage() (*types.ImageInspect, error) {
imageName, err := e.expandImageName(e.Build.Image.Name, []string{})
if err != nil {
return nil, err
}
// Fetch image
image, err := e.pullManager.GetDockerImage(imageName)
if err != nil {
return nil, err
}
return image, nil
}
func fakeContainer(id string, names ...string) *types.Container {
return &types.Container{ID: id, Names: names}
}
func (e *executor) parseDeviceString(deviceString string) (device container.DeviceMapping, err error) {
// Split the device string PathOnHost[:PathInContainer[:CgroupPermissions]]
parts := strings.Split(deviceString, ":")
if len(parts) > 3 {
err = fmt.Errorf("too many colons")
return
}
device.PathOnHost = parts[0]
// Optional container path
if len(parts) >= 2 {
device.PathInContainer = parts[1]
} else {
// default: device at same path in container
device.PathInContainer = device.PathOnHost
}
// Optional permissions
if len(parts) >= 3 {
device.CgroupPermissions = parts[2]
} else {
// default: rwm, just like 'docker run'
device.CgroupPermissions = "rwm"
}
return
}
func (e *executor) bindDevices() (err error) {
for _, deviceString := range e.Config.Docker.Devices {
device, err := e.parseDeviceString(deviceString)
if err != nil {
err = fmt.Errorf("failed to parse device string %q: %w", deviceString, err)
return err
}
e.devices = append(e.devices, device)
}
return nil
}
func (e *executor) bindDeviceRequests() error {
if e.Config.Docker.Gpus == "" {
return nil
}
var gpus opts.GpuOpts
err := gpus.Set(e.Config.Docker.Gpus)
if err != nil {
return fmt.Errorf("parsing deviceRequest string %q: %w", e.Config.Docker.Gpus, err)
}
e.deviceRequests = gpus.Value()
return nil
}
func (e *executor) createService(
serviceIndex int,
service, version, image string,
serviceDefinition common.Image,
linkNames []string,
) (*types.Container, error) {
if service == "" {
return nil, fmt.Errorf("invalid service name: %s", serviceDefinition.Name)
}
if e.volumesManager == nil {
return nil, errVolumesManagerUndefined
}
e.Println("Starting service", service+":"+version, "...")
serviceImage, err := e.pullManager.GetDockerImage(image)
if err != nil {
return nil, err
}
serviceSlug := strings.ReplaceAll(service, "/", "__")
containerName := fmt.Sprintf("%s-%s-%d", e.getProjectUniqRandomizedName(), serviceSlug, serviceIndex)
// this will fail potentially some builds if there's name collision
_ = e.removeContainer(e.Context, containerName)
labels := map[string]string{
"type": labelServiceType,
"service": service,
"service.version": version,
}
config := &container.Config{
Image: serviceImage.ID,
Labels: e.labeler.Labels(labels),
Env: append(e.getServiceVariables(), e.BuildShell.Environment...),
}
if len(serviceDefinition.Command) > 0 {
config.Cmd = serviceDefinition.Command
}
config.Entrypoint = e.overwriteEntrypoint(&serviceDefinition)
hostConfig := e.createHostConfigForService()
networkConfig := e.networkConfig(linkNames)
e.Debugln("Creating service container", containerName, "...")
resp, err := e.client.ContainerCreate(e.Context, config, hostConfig, networkConfig, containerName)
if err != nil {
return nil, err
}
e.Debugln(fmt.Sprintf("Starting service container %s (%s)...", containerName, resp.ID))
err = e.client.ContainerStart(e.Context, resp.ID, types.ContainerStartOptions{})
if err != nil {
e.temporary = append(e.temporary, resp.ID)
return nil, err
}
return fakeContainer(resp.ID, containerName), nil
}
func (e *executor) createHostConfigForService() *container.HostConfig {
return &container.HostConfig{
DNS: e.Config.Docker.DNS,
DNSSearch: e.Config.Docker.DNSSearch,
RestartPolicy: neverRestartPolicy,
ExtraHosts: e.Config.Docker.ExtraHosts,
Privileged: e.Config.Docker.Privileged,
UsernsMode: container.UsernsMode(e.Config.Docker.UsernsMode),
NetworkMode: e.networkMode,
Binds: e.volumesManager.Binds(),
ShmSize: e.Config.Docker.ShmSize,
Tmpfs: e.Config.Docker.ServicesTmpfs,
LogConfig: container.LogConfig{
Type: "json-file",
},
}
}
func (e *executor) networkConfig(aliases []string) *network.NetworkingConfig {
if e.networkMode.UserDefined() == "" {
return &network.NetworkingConfig{}
}
return &network.NetworkingConfig{
EndpointsConfig: map[string]*network.EndpointSettings{
e.networkMode.UserDefined(): {Aliases: aliases},
},
}
}
func (e *executor) getProjectUniqRandomizedName() string {
if e.projectUniqRandomizedName == "" {
uuid, _ := helpers.GenerateRandomUUID(8)
e.projectUniqRandomizedName = fmt.Sprintf("%s-%s", e.Build.ProjectUniqueName(), uuid)
}
return e.projectUniqRandomizedName
}
func (e *executor) getServicesDefinitions() (common.Services, error) {
var internalServiceImages []string
serviceDefinitions := common.Services{}
for _, service := range e.Config.Docker.Services {
internalServiceImages = append(internalServiceImages, service.Name)
serviceDefinitions = append(serviceDefinitions, service.ToImageDefinition())
}
for _, service := range e.Build.Services {
serviceName := e.Build.GetAllVariables().ExpandValue(service.Name)
err := e.verifyAllowedImage(serviceName, "services", e.Config.Docker.AllowedServices, internalServiceImages)
if err != nil {
return nil, err
}
service.Name = serviceName
serviceDefinitions = append(serviceDefinitions, service)
}
return serviceDefinitions, nil
}
func (e *executor) waitForServices() {
waitForServicesTimeout := e.Config.Docker.WaitForServicesTimeout
if waitForServicesTimeout == 0 {
waitForServicesTimeout = common.DefaultWaitForServicesTimeout
}
// wait for all services to came up
if waitForServicesTimeout > 0 && len(e.services) > 0 {
e.Println("Waiting for services to be up and running...")
wg := sync.WaitGroup{}
for _, service := range e.services {
wg.Add(1)
go func(service *types.Container) {
_ = e.waitForServiceContainer(service, time.Duration(waitForServicesTimeout)*time.Second)
wg.Done()
}(service)
}
wg.Wait()
}
}
func (e *executor) buildServiceLinks(linksMap map[string]*types.Container) (links []string) {
for linkName, linkee := range linksMap {
newContainer, err := e.client.ContainerInspect(e.Context, linkee.ID)
if err != nil {
continue
}
if newContainer.State.Running {
links = append(links, linkee.ID+":"+linkName)
}
}
return
}
func (e *executor) createFromServiceDefinition(
serviceIndex int,
serviceDefinition common.Image,
linksMap map[string]*types.Container,
) error {
var container *types.Container
serviceMeta := services.SplitNameAndVersion(serviceDefinition.Name)
if serviceDefinition.Alias != "" {
serviceMeta.Aliases = append(serviceMeta.Aliases, serviceDefinition.Alias)
}
for _, linkName := range serviceMeta.Aliases {
if linksMap[linkName] != nil {
e.Warningln("Service", serviceDefinition.Name, "is already created. Ignoring.")
continue
}
// Create service if not yet created
if container == nil {
var err error
container, err = e.createService(
serviceIndex,
serviceMeta.Service,
serviceMeta.Version,
serviceMeta.ImageName,
serviceDefinition,
serviceMeta.Aliases,
)
if err != nil {
return err
}
e.Debugln("Created service", serviceDefinition.Name, "as", container.ID)
e.services = append(e.services, container)
e.temporary = append(e.temporary, container.ID)
}
linksMap[linkName] = container
}
return nil
}
func (e *executor) createBuildNetwork() error {
if e.networksManager == nil {
return errNetworksManagerUndefined
}
networkMode, err := e.networksManager.Create(e.Context, e.Config.Docker.NetworkMode)
if err != nil {
return err
}
e.networkMode = networkMode
return nil
}
func (e *executor) cleanupNetwork(ctx context.Context) error {
if e.networksManager == nil {
return errNetworksManagerUndefined
}
if e.networkMode.UserDefined() == "" {
return nil
}
inspectResponse, err := e.networksManager.Inspect(ctx)
if err != nil {
e.Errorln("network inspect returned error ", err)
return nil
}
for id := range inspectResponse.Containers {
e.Debugln("Removing Container", id, "...")
err = e.removeContainer(ctx, id)
if err != nil {
e.Errorln("remove container returned error ", err)
}
}
return e.networksManager.Cleanup(ctx)
}
func (e *executor) createServices() (err error) {
e.SetCurrentStage(ExecutorStageCreatingServices)
e.Debugln("Creating services...")
servicesDefinitions, err := e.getServicesDefinitions()
if err != nil {
return
}
linksMap := make(map[string]*types.Container)
for index, serviceDefinition := range servicesDefinitions {
err = e.createFromServiceDefinition(index, serviceDefinition, linksMap)
if err != nil {
return
}
}
e.waitForServices()
if e.networkMode.IsBridge() || e.networkMode.NetworkName() == "" {
e.Debugln("Building service links...")
e.links = e.buildServiceLinks(linksMap)
}
return
}
func (e *executor) createContainer(
containerType string,
imageDefinition common.Image,
cmd []string,
allowedInternalImages []string,
) (*types.ContainerJSON, error) {
if e.volumesManager == nil {
return nil, errVolumesManagerUndefined
}
image, err := e.expandAndGetDockerImage(imageDefinition.Name, allowedInternalImages)
if err != nil {
return nil, err
}
hostname := e.Config.Docker.Hostname
if hostname == "" {
hostname = e.Build.ProjectUniqueName()
}
// Always create unique, but sequential name
containerIndex := len(e.builds)
containerName := e.getProjectUniqRandomizedName() + "-" + containerType + "-" + strconv.Itoa(containerIndex)
config := e.createContainerConfig(containerType, imageDefinition, image.ID, hostname, cmd)
hostConfig, err := e.createHostConfig()
if err != nil {
return nil, err
}
aliases := []string{"build", containerName}
networkConfig := e.networkConfig(aliases)
// this will fail potentially some builds if there's name collision
_ = e.removeContainer(e.Context, containerName)
e.Debugln("Creating container", containerName, "...")
resp, err := e.client.ContainerCreate(e.Context, config, hostConfig, networkConfig, containerName)
if err != nil {
if resp.ID != "" {
e.temporary = append(e.temporary, resp.ID)
}
return nil, err
}
inspect, err := e.client.ContainerInspect(e.Context, resp.ID)
if err != nil {
e.temporary = append(e.temporary, resp.ID)
return nil, err
}
e.builds = append(e.builds, resp.ID)
e.temporary = append(e.temporary, resp.ID)
return &inspect, nil
}
func (e *executor) createContainerConfig(
containerType string,
imageDefinition common.Image,
imageID string,
hostname string,
cmd []string,
) *container.Config {
config := &container.Config{
Image: imageID,
Hostname: hostname,
Cmd: cmd,
Labels: e.labeler.Labels(map[string]string{"type": containerType}),
Tty: false,
AttachStdin: true,
AttachStdout: true,
AttachStderr: true,
OpenStdin: true,
StdinOnce: true,
Env: append(e.Build.GetAllVariables().StringList(), e.BuildShell.Environment...),
}
config.Entrypoint = e.overwriteEntrypoint(&imageDefinition)
return config
}
func (e *executor) createHostConfig() (*container.HostConfig, error) {
nanoCPUs, err := e.Config.Docker.GetNanoCPUs()
if err != nil {
return nil, err
}
return &container.HostConfig{
Resources: container.Resources{
Memory: e.Config.Docker.GetMemory(),
MemorySwap: e.Config.Docker.GetMemorySwap(),
MemoryReservation: e.Config.Docker.GetMemoryReservation(),
CpusetCpus: e.Config.Docker.CPUSetCPUs,
CPUShares: e.Config.Docker.CPUShares,
NanoCPUs: nanoCPUs,
Devices: e.devices,
DeviceRequests: e.deviceRequests,
OomKillDisable: e.Config.Docker.GetOomKillDisable(),
},
DNS: e.Config.Docker.DNS,
DNSSearch: e.Config.Docker.DNSSearch,
Runtime: e.Config.Docker.Runtime,
Privileged: e.Config.Docker.Privileged,
UsernsMode: container.UsernsMode(e.Config.Docker.UsernsMode),
CapAdd: e.Config.Docker.CapAdd,
CapDrop: e.Config.Docker.CapDrop,
SecurityOpt: e.Config.Docker.SecurityOpt,
RestartPolicy: neverRestartPolicy,
ExtraHosts: e.Config.Docker.ExtraHosts,
NetworkMode: e.networkMode,
Links: append(e.Config.Docker.Links, e.links...),
Binds: e.volumesManager.Binds(),
OomScoreAdj: e.Config.Docker.OomScoreAdjust,
ShmSize: e.Config.Docker.ShmSize,
VolumeDriver: e.Config.Docker.VolumeDriver,
VolumesFrom: e.Config.Docker.VolumesFrom,
LogConfig: container.LogConfig{
Type: "json-file",
},
Tmpfs: e.Config.Docker.Tmpfs,
Sysctls: e.Config.Docker.SysCtls,
}, nil
}
func (e *executor) startAndWatchContainer(ctx context.Context, id string, input io.Reader) error {
dockerExec := exec.NewDocker(e.client, e.waiter, e.Build.Log())
return dockerExec.Exec(ctx, id, input, e.Trace)
}
func (e *executor) removeContainer(ctx context.Context, id string) error {
e.Debugln("Removing container", id)
e.disconnectNetwork(ctx, id)
options := types.ContainerRemoveOptions{
RemoveVolumes: true,
Force: true,
}
err := e.client.ContainerRemove(ctx, id, options)
if err != nil {
e.Debugln("Removing container", id, "finished with error", err)
return err
}
e.Debugln("Removed container", id)
return nil
}
func (e *executor) disconnectNetwork(ctx context.Context, id string) {
e.Debugln("Disconnecting container", id, "from networks")
netList, err := e.client.NetworkList(ctx, types.NetworkListOptions{})
if err != nil {
e.Debugln("Can't get network list. ListNetworks exited with", err)
return
}
for _, network := range netList {
for _, pluggedContainer := range network.Containers {
if id == pluggedContainer.Name {
err = e.client.NetworkDisconnect(ctx, network.ID, id, true)
if err != nil {
e.Warningln(
"Can't disconnect possibly zombie container",
pluggedContainer.Name,
"from network",
network.Name,
"->",
err,
)
} else {
e.Warningln(
"Possibly zombie container",
pluggedContainer.Name,
"is disconnected from network",
network.Name,
)
}
break
}
}
}
}
func (e *executor) verifyAllowedImage(image, optionName string, allowedImages, internalImages []string) error {
options := common.VerifyAllowedImageOptions{
Image: image,
OptionName: optionName,
AllowedImages: allowedImages,
InternalImages: internalImages,
}
return common.VerifyAllowedImage(options, e.BuildLogger)
}
func (e *executor) expandImageName(imageName string, allowedInternalImages []string) (string, error) {
if imageName != "" {
image := e.Build.GetAllVariables().ExpandValue(imageName)
allowedInternalImages = append(allowedInternalImages, e.Config.Docker.Image)
err := e.verifyAllowedImage(image, "images", e.Config.Docker.AllowedImages, allowedInternalImages)
if err != nil {
return "", err
}
return image, nil
}
if e.Config.Docker.Image == "" {
return "", errors.New("no Docker image specified to run the build in")
}
return e.Config.Docker.Image, nil
}
func (e *executor) overwriteEntrypoint(image *common.Image) []string {
if len(image.Entrypoint) > 0 {
if !e.Config.Docker.DisableEntrypointOverwrite {
return image.Entrypoint
}
e.Warningln("Entrypoint override disabled")
}
return nil
}
func (e *executor) connectDocker() error {
client, err := docker.New(e.Config.Docker.Credentials, "")
if err != nil {
return err
}
e.client = client
e.info, err = client.Info(e.Context)
if err != nil {
return err
}
err = e.validateOSType()
if err != nil {
return err
}
e.waiter = wait.NewDockerKillWaiter(e.client)
return err
}
// validateOSType checks if the ExecutorOptions metadata matches with the docker
// info response.
func (e *executor) validateOSType() error {
executorOSType := e.ExecutorOptions.Metadata[metadataOSType]
if executorOSType == "" {
return common.MakeBuildError("%s does not have any OSType specified", e.Config.Executor)
}
if executorOSType != e.info.OSType {
return common.MakeBuildError(
"executor requires OSType=%s, but Docker Engine supports only OSType=%s",
executorOSType, e.info.OSType,
)
}
return nil
}
func (e *executor) createDependencies() error {
createDependenciesStrategy := []func() error{
e.createLabeler,
e.createNetworksManager,
e.createBuildNetwork,
e.createPullManager,
e.bindDevices,
e.bindDeviceRequests,
e.createVolumesManager,
e.createVolumes,
e.createBuildVolume,
e.createServices,
}
for _, setup := range createDependenciesStrategy {
err := setup()
if err != nil {
return err
}
}
return nil
}
func (e *executor) createVolumes() error {
e.SetCurrentStage(ExecutorStageCreatingUserVolumes)
e.Debugln("Creating user-defined volumes...")
if e.volumesManager == nil {
return errVolumesManagerUndefined
}
for _, volume := range e.Config.Docker.Volumes {
err := e.volumesManager.Create(e.Context, volume)
if errors.Is(err, volumes.ErrCacheVolumesDisabled) {
e.Warningln(fmt.Sprintf(
"Container based cache volumes creation is disabled. Will not create volume for %q",
volume,
))
continue
}
if err != nil {
return err
}
}
return nil
}
func (e *executor) createBuildVolume() error {
e.SetCurrentStage(ExecutorStageCreatingBuildVolumes)
e.Debugln("Creating build volume...")
if e.volumesManager == nil {
return errVolumesManagerUndefined
}
jobsDir := e.Build.RootDir
var err error
if e.Build.GetGitStrategy() == common.GitFetch {
err = e.volumesManager.Create(e.Context, jobsDir)
if err == nil {
return nil
}
if errors.Is(err, volumes.ErrCacheVolumesDisabled) {
err = e.volumesManager.CreateTemporary(e.Context, jobsDir)
}
} else {
err = e.volumesManager.CreateTemporary(e.Context, jobsDir)
}
if err != nil {
var volDefinedErr *volumes.ErrVolumeAlreadyDefined
if !errors.As(err, &volDefinedErr) {
return err
}
}
return nil
}
func (e *executor) Prepare(options common.ExecutorPrepareOptions) error {
e.SetCurrentStage(ExecutorStagePrepare)
if options.Config.Docker == nil {
return errors.New("missing docker configuration")
}
e.AbstractExecutor.PrepareConfiguration(options)
err := e.connectDocker()
if err != nil {
return err
}
e.helperImageInfo, err = e.prepareHelperImage()
if err != nil {
return err
}
err = e.prepareBuildsDir(options)
if err != nil {
return err
}
err = e.AbstractExecutor.PrepareBuildAndShell()
if err != nil {
return err
}
if e.BuildShell.PassFile {
return errors.New("docker doesn't support shells that require script file")
}
imageName, err := e.expandImageName(e.Build.Image.Name, []string{})
if err != nil {
return err
}
e.Println("Using Docker executor with image", imageName, "...")
err = e.createDependencies()
if err != nil {
return err
}
return nil
}
func (e *executor) prepareHelperImage() (helperimage.Info, error) {
return helperimage.Get(common.REVISION, helperimage.Config{
OSType: e.info.OSType,
Architecture: e.info.Architecture,
OperatingSystem: e.info.OperatingSystem,
Shell: e.Config.Shell,
GitLabRegistry: e.Build.IsFeatureFlagOn(featureflags.GitLabRegistryHelperImage),
Flavor: e.Config.Docker.HelperImageFlavor,
})
}
func (e *executor) prepareBuildsDir(options common.ExecutorPrepareOptions) error {
if e.volumeParser == nil {
return common.MakeBuildError("missing volume parser")
}
isHostMounted, err := volumes.IsHostMountedVolume(e.volumeParser, e.RootDir(), options.Config.Docker.Volumes...)
if err != nil {
return &common.BuildError{Inner: err}
}
// We need to set proper value for e.SharedBuildsDir because
// it's required to properly start the job, what is done inside of
// e.AbstractExecutor.Prepare()
// And a started job is required for Volumes Manager to work, so it's
// done before the manager is even created.
if isHostMounted {
e.SharedBuildsDir = true
}
return nil
}
func (e *executor) Cleanup() {
e.SetCurrentStage(ExecutorStageCleanup)
var wg sync.WaitGroup
ctx, cancel := context.WithTimeout(context.Background(), dockerCleanupTimeout)
defer cancel()
remove := func(id string) {
wg.Add(1)
go func() {
_ = e.removeContainer(ctx, id)
wg.Done()
}()
}
for _, temporaryID := range e.temporary {
remove(temporaryID)
}
wg.Wait()
err := e.cleanupVolume(ctx)
if err != nil {
volumeLogger := e.WithFields(logrus.Fields{
"error": err,
})
volumeLogger.Errorln("Failed to cleanup volumes")
}
err = e.cleanupNetwork(ctx)
if err != nil {
networkLogger := e.WithFields(logrus.Fields{
"network": e.networkMode.NetworkName(),
"error": err,
})
networkLogger.Errorln("Failed to remove network for build")
}
if e.client != nil {
_ = e.client.Close()
}
e.AbstractExecutor.Cleanup()
}
func (e *executor) cleanupVolume(ctx context.Context) error {
if e.volumesManager == nil {
e.Debugln("Volumes manager is empty, skipping volumes cleanup")
return nil
}
err := e.volumesManager.RemoveTemporary(ctx)
if err != nil {
return fmt.Errorf("remove temporary volumes: %w", err)
}
return nil
}
type serviceHealthCheckError struct {
Inner error
Logs string
}
func (e *serviceHealthCheckError) Error() string {
if e.Inner == nil {
return "serviceHealthCheckError"
}
return e.Inner.Error()
}
func (e *executor) runServiceHealthCheckContainer(service *types.Container, timeout time.Duration) error {
waitImage, err := e.getPrebuiltImage()
if err != nil {
return fmt.Errorf("getPrebuiltImage: %w", err)
}
containerName := service.Names[0] + "-wait-for-service"
environment, err := e.addServiceHealthCheckEnvironment(service)
if err != nil {
return err
}
cmd := []string{"gitlab-runner-helper", "health-check"}
config := e.createConfigForServiceHealthCheckContainer(service, cmd, waitImage, environment)
hostConfig := e.createHostConfigForServiceHealthCheck(service)
e.Debugln(fmt.Sprintf("Creating service healthcheck container %s...", containerName))
resp, err := e.client.ContainerCreate(e.Context, config, hostConfig, nil, containerName)
if err != nil {
return fmt.Errorf("create service container: %w", err)
}
defer func() { _ = e.removeContainer(e.Context, resp.ID) }()
e.Debugln(fmt.Sprintf("Starting service healthcheck container %s (%s)...", containerName, resp.ID))
err = e.client.ContainerStart(e.Context, resp.ID, types.ContainerStartOptions{})
if err != nil {
return fmt.Errorf("start service container: %w", err)
}
ctx, cancel := context.WithTimeout(e.Context, timeout)
defer cancel()
err = e.waiter.Wait(ctx, resp.ID)
if err == nil {
return nil
}
if errors.Is(err, context.DeadlineExceeded) {
err = fmt.Errorf("service %q timeout", containerName)
} else {
err = fmt.Errorf("service %q health check: %w", containerName, err)
}
return &serviceHealthCheckError{
Inner: err,
Logs: e.readContainerLogs(resp.ID),
}
}
func (e *executor) createConfigForServiceHealthCheckContainer(
service *types.Container,
cmd []string,
waitImage *types.ImageInspect,
environment []string,
) *container.Config {
return &container.Config{
Cmd: cmd,
Image: waitImage.ID,
Labels: e.labeler.Labels(map[string]string{"type": labelWaitType, "wait": service.ID}),
Env: environment,
}
}
func (e *executor) createHostConfigForServiceHealthCheck(service *types.Container) *container.HostConfig {
return &container.HostConfig{
RestartPolicy: neverRestartPolicy,
Links: []string{service.Names[0] + ":service"},
NetworkMode: e.networkMode,
LogConfig: container.LogConfig{
Type: "json-file",
},
}
}
// addServiceHealthCheckEnvironment returns environment variables mimicing
// the legacy container links networking feature of Docker, where environment
// variables are provided with the hostname and port of the linked service our
// health check is performed against.
//
// The hostname we provide is the container's short ID (the first 12 characters
// of a full container ID). The short ID, as opposed to the full ID, is
// internally resolved to the container's IP address by Docker's built-in DNS
// service.
//
// The legacy container links (https://docs.docker.com/network/links/) network
// feature is deprecated. When we remove support for links, the healthcheck
// system can be updated to no longer rely on environment variables
func (e *executor) addServiceHealthCheckEnvironment(service *types.Container) ([]string, error) {
environment := []string{}
if e.networkMode.UserDefined() != "" {
environment = append(environment, "WAIT_FOR_SERVICE_TCP_ADDR="+service.ID[:12])
ports, err := e.getContainerExposedPorts(service)
if err != nil {
return nil, fmt.Errorf("get container exposed ports: %v", err)
}
if len(ports) == 0 {
return nil, fmt.Errorf("service %q has no exposed ports", service.Names[0])
}
environment = append(environment, fmt.Sprintf("WAIT_FOR_SERVICE_TCP_PORT=%d", ports[0]))
}
return environment, nil
}
func (e *executor) getContainerExposedPorts(container *types.Container) ([]int, error) {
var ports []int
inspect, err := e.client.ContainerInspect(e.Context, container.ID)
if err != nil {
return nil, err
}
for port := range inspect.Config.ExposedPorts {
start, _, err := port.Range()
if err == nil && port.Proto() == "tcp" {
ports = append(ports, start)
}
}
sort.Ints(ports)
return ports, nil
}
func (e *executor) waitForServiceContainer(service *types.Container, timeout time.Duration) error {
err := e.runServiceHealthCheckContainer(service, timeout)
if err == nil {
return nil
}
var buffer bytes.Buffer
buffer.WriteString("\n")
buffer.WriteString(
helpers.ANSI_YELLOW + "*** WARNING:" + helpers.ANSI_RESET + " Service " + service.Names[0] +
" probably didn't start properly.\n")
buffer.WriteString("\n")
buffer.WriteString("Health check error:\n")
buffer.WriteString(strings.TrimSpace(err.Error()))
buffer.WriteString("\n")
if healtCheckErr, ok := err.(*serviceHealthCheckError); ok {
buffer.WriteString("\n")
buffer.WriteString("Health check container logs:\n")
buffer.WriteString(healtCheckErr.Logs)
buffer.WriteString("\n")
}
buffer.WriteString("\n")
buffer.WriteString("Service container logs:\n")
buffer.WriteString(e.readContainerLogs(service.ID))
buffer.WriteString("\n")
buffer.WriteString("\n")
buffer.WriteString(helpers.ANSI_YELLOW + "*********" + helpers.ANSI_RESET + "\n")
buffer.WriteString("\n")
_, _ = io.Copy(e.Trace, &buffer)
return err
}
func (e *executor) readContainerLogs(containerID string) string {
var containerBuffer bytes.Buffer
options := types.ContainerLogsOptions{
ShowStdout: true,
ShowStderr: true,
Timestamps: true,
}
hijacked, err := e.client.ContainerLogs(e.Context, containerID, options)
if err != nil {
return strings.TrimSpace(err.Error())
}
defer func() { _ = hijacked.Close() }()
_, _ = stdcopy.StdCopy(&containerBuffer, &containerBuffer, hijacked)
containerLog := containerBuffer.String()
return strings.TrimSpace(containerLog)
}
package docker
import (
"bytes"
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/docker/docker/api/types"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/exec"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/user"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/permission"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
)
type commandExecutor struct {
executor
buildContainer *types.ContainerJSON
lock sync.Mutex
terminalWaitForContainerTimeout time.Duration
}
func (s *commandExecutor) getBuildContainer() *types.ContainerJSON {
s.lock.Lock()
defer s.lock.Unlock()
return s.buildContainer
}
func (s *commandExecutor) Prepare(options common.ExecutorPrepareOptions) error {
err := s.executor.Prepare(options)
if err != nil {
return err
}
s.Debugln("Starting Docker command...")
if len(s.BuildShell.DockerCommand) == 0 {
return errors.New("script is not compatible with Docker")
}
_, err = s.getPrebuiltImage()
if err != nil {
return err
}
_, err = s.getBuildImage()
if err != nil {
return err
}
if s.isUmaskDisabled() {
s.Println("Not using umask - FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR is set!")
}
return nil
}
func (s *commandExecutor) isUmaskDisabled() bool {
// Not usable with docker-windows executor
if s.AbstractExecutor.ExecutorOptions.Metadata[metadataOSType] == osTypeWindows {
return false
}
if !s.Build.IsFeatureFlagOn(featureflags.DisableUmaskForDockerExecutor) {
return false
}
return true
}
func (s *commandExecutor) Run(cmd common.ExecutorCommand) error {
maxAttempts, err := s.Build.GetExecutorJobSectionAttempts()
if err != nil {
return fmt.Errorf("getting job section attempts: %w", err)
}
var runErr error
for attempts := 1; attempts <= maxAttempts; attempts++ {
if attempts > 1 {
s.Infoln(fmt.Sprintf("Retrying %s", cmd.Stage))
}
ctr, err := s.getContainer(cmd)
if err != nil {
return err
}
s.Debugln("Executing on", ctr.Name, "the", cmd.Script)
s.SetCurrentStage(ExecutorStageRun)
runErr = s.startAndWatchContainer(cmd.Context, ctr.ID, bytes.NewBufferString(cmd.Script))
if !docker.IsErrNotFound(runErr) {
return runErr
}
s.Errorln(fmt.Sprintf("Container %q not found or removed. Will retry...", ctr.ID))
}
if runErr != nil && maxAttempts > 1 {
s.Errorln("Execution attempts exceeded")
}
return runErr
}
func (s *commandExecutor) getContainer(cmd common.ExecutorCommand) (*types.ContainerJSON, error) {
if cmd.Predefined {
return s.requestNewPredefinedContainer()
}
return s.requestBuildContainer()
}
func (s *commandExecutor) requestNewPredefinedContainer() (*types.ContainerJSON, error) {
prebuildImage, err := s.getPrebuiltImage()
if err != nil {
return nil, err
}
buildImage := common.Image{
Name: prebuildImage.ID,
}
containerJSON, err := s.createContainer("predefined", buildImage, s.getHelperImageCmd(), []string{prebuildImage.ID})
if err != nil {
return nil, err
}
return containerJSON, err
}
func (s *commandExecutor) getHelperImageCmd() []string {
if s.isUmaskDisabled() {
return []string{"/bin/bash"}
}
return s.helperImageInfo.Cmd
}
func (s *commandExecutor) requestBuildContainer() (*types.ContainerJSON, error) {
s.lock.Lock()
defer s.lock.Unlock()
if s.buildContainer != nil {
_, inspectErr := s.client.ContainerInspect(s.Context, s.buildContainer.ID)
if inspectErr == nil {
return s.buildContainer, nil
}
if !docker.IsErrNotFound(inspectErr) {
s.Warningln("Failed to inspect build container", s.buildContainer.ID, inspectErr.Error())
}
}
var err error
s.buildContainer, err = s.createContainer("build", s.Build.Image, s.BuildShell.DockerCommand, []string{})
if err != nil {
return nil, err
}
err = s.changeFilesOwnership()
if err != nil {
return nil, err
}
return s.buildContainer, nil
}
func (s *commandExecutor) changeFilesOwnership() error {
if !s.isUmaskDisabled() {
return nil
}
dockerExec := exec.NewDocker(s.client, s.waiter, s.Build.Log())
inspect := user.NewInspect(s.client, dockerExec)
imageSHA := s.buildContainer.Image
imageName := s.Build.Image.Name
log := s.Build.Log().WithFields(logrus.Fields{
"imageSHA": imageSHA,
"imageName": imageName,
})
log.Debug("Checking if image runs with root user")
usesRoot, err := inspect.IsRoot(s.Context, imageSHA)
if err != nil {
return fmt.Errorf("checking if image %q runs as root: %w", imageName, err)
}
if usesRoot {
log.Debug("Image uses root user")
return nil
}
log.Debug("Image doesn't use root user")
uid, gid, err := getUIDandGID(s.Context, log, inspect, s.buildContainer.ID, imageSHA)
if err != nil {
return err
}
if uid == 0 {
return nil
}
return s.executeChown(dockerExec, uid, gid)
}
func getUIDandGID(
ctx context.Context,
log logrus.FieldLogger,
inspect user.Inspect,
buildContainerID string,
imageSHA string,
) (int, int, error) {
containerLog := log.WithField("container", buildContainerID)
containerLog.Debug("Getting the UID of the container")
uid, err := inspect.UID(ctx, buildContainerID)
if err != nil {
return 0, 0, fmt.Errorf("checking %q image's UID: %w", imageSHA, err)
}
containerLog.Debugf("Container UID=%d", uid)
containerLog.Debug("Getting the GID of the container")
gid, err := inspect.GID(ctx, buildContainerID)
if err != nil {
return 0, 0, fmt.Errorf("checking %q image's GID: %w", imageSHA, err)
}
containerLog.Debugf("Container GID=%d", gid)
return uid, gid, err
}
func (s *commandExecutor) executeChown(dockerExec exec.Docker, uid int, gid int) error {
c, err := s.requestNewPredefinedContainer()
if err != nil {
return fmt.Errorf("requesting new predefined container: %w", err)
}
err = s.executeChownOnDir(c, dockerExec, uid, gid, s.Build.FullProjectDir())
if err != nil {
return err
}
err = s.executeChownOnDir(c, dockerExec, uid, gid, s.Build.TmpProjectDir())
if err != nil {
return err
}
return nil
}
func (s *commandExecutor) executeChownOnDir(
c *types.ContainerJSON,
dockerExec exec.Docker,
uid int,
gid int,
dir string,
) error {
s.Println(fmt.Sprintf("Changing ownership of files at %q to %d:%d", dir, uid, gid))
input := bytes.NewBufferString(fmt.Sprintf("chown -RP -- %d:%d %q", uid, gid, dir))
output := new(bytes.Buffer)
err := dockerExec.Exec(s.Context, c.ID, input, output)
log := s.Build.Log().WithField("updatedDir", dir)
log.WithField("output", output.String()).Debug("Changing ownership of files")
if err != nil {
log.WithError(err).Error("Failed to change ownership of files")
}
return nil
}
func (s *commandExecutor) GetMetricsSelector() string {
return fmt.Sprintf("instance=%q", s.executor.info.Name)
}
func init() {
options := executors.ExecutorOptions{
DefaultCustomBuildsDirEnabled: true,
DefaultBuildsDir: "/builds",
DefaultCacheDir: "/cache",
SharedBuildsDir: false,
Shell: common.ShellScriptInfo{
Shell: "bash",
Type: common.NormalShell,
RunnerCommand: "/usr/bin/gitlab-runner-helper",
},
ShowHostname: true,
Metadata: map[string]string{
metadataOSType: osTypeLinux,
},
}
creator := func() common.Executor {
e := &commandExecutor{
executor: executor{
AbstractExecutor: executors.AbstractExecutor{
ExecutorOptions: options,
},
volumeParser: parser.NewLinuxParser(),
},
}
e.newVolumePermissionSetter = func() (permission.Setter, error) {
helperImage, err := e.getPrebuiltImage()
if err != nil {
return nil, err
}
return permission.NewDockerLinuxSetter(e.client, e.Build.Log(), helperImage), nil
}
e.SetCurrentStage(common.ExecutorStageCreated)
return e
}
featuresUpdater := func(features *common.FeaturesInfo) {
features.Variables = true
features.Image = true
features.Services = true
features.Session = true
features.Terminal = true
}
common.RegisterExecutorProvider("docker", executors.DefaultExecutorProvider{
Creator: creator,
FeaturesUpdater: featuresUpdater,
ConfigUpdater: configUpdater,
DefaultShellName: options.Shell.Shell,
})
}
package docker
import (
"errors"
"github.com/docker/docker/api/types"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser"
"gitlab.com/gitlab-org/gitlab-runner/helpers/ssh"
)
type sshExecutor struct {
executor
sshCommand ssh.Client
}
func (s *sshExecutor) Prepare(options common.ExecutorPrepareOptions) error {
err := s.executor.Prepare(options)
if err != nil {
return err
}
s.Warningln(
"Since GitLab Runner 10.0 docker-ssh and docker-ssh+machine executors " +
"are marked as DEPRECATED and will be removed in one of the upcoming releases")
if s.Config.SSH == nil {
return errors.New("missing SSH configuration")
}
s.Debugln("Starting SSH command...")
// Start build container which will run actual build
container, err := s.createContainer("build", s.Build.Image, []string{}, []string{})
if err != nil {
return err
}
s.Debugln("Starting container", container.ID, "...")
err = s.client.ContainerStart(s.Context, container.ID, types.ContainerStartOptions{})
if err != nil {
return err
}
containerData, err := s.client.ContainerInspect(s.Context, container.ID)
if err != nil {
return err
}
// Create SSH command
s.sshCommand = ssh.Client{
Config: *s.Config.SSH,
Stdout: s.Trace,
Stderr: s.Trace,
}
s.sshCommand.Host = containerData.NetworkSettings.IPAddress
s.Debugln("Connecting to SSH server...")
err = s.sshCommand.Connect()
if err != nil {
return err
}
return nil
}
func (s *sshExecutor) Run(cmd common.ExecutorCommand) error {
s.SetCurrentStage(ExecutorStageRun)
err := s.sshCommand.Run(cmd.Context, ssh.Command{
Environment: s.BuildShell.Environment,
Command: s.BuildShell.GetCommandWithArguments(),
Stdin: cmd.Script,
})
if exitError, ok := err.(*ssh.ExitError); ok {
exitCode := exitError.ExitCode()
err = &common.BuildError{Inner: err, ExitCode: exitCode}
}
return err
}
func (s *sshExecutor) Cleanup() {
s.sshCommand.Cleanup()
s.executor.Cleanup()
}
func init() {
options := executors.ExecutorOptions{
DefaultCustomBuildsDirEnabled: true,
DefaultBuildsDir: "builds",
DefaultCacheDir: "cache",
SharedBuildsDir: false,
Shell: common.ShellScriptInfo{
Shell: "bash",
Type: common.LoginShell,
RunnerCommand: "gitlab-runner",
},
ShowHostname: true,
Metadata: map[string]string{
metadataOSType: osTypeLinux,
},
}
creator := func() common.Executor {
e := &sshExecutor{
executor: executor{
AbstractExecutor: executors.AbstractExecutor{
ExecutorOptions: options,
},
volumeParser: parser.NewLinuxParser(),
},
}
e.SetCurrentStage(common.ExecutorStageCreated)
return e
}
featuresUpdater := func(features *common.FeaturesInfo) {
features.Variables = true
features.Image = true
features.Services = true
}
common.RegisterExecutorProvider("docker-ssh", executors.DefaultExecutorProvider{
Creator: creator,
FeaturesUpdater: featuresUpdater,
ConfigUpdater: configUpdater,
DefaultShellName: options.Shell.Shell,
})
}
package exec
import (
"context"
"errors"
"io"
"net"
"github.com/docker/docker/api/types"
"github.com/docker/docker/pkg/stdcopy"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/wait"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
)
// conn is an interface wrapper used to generate mocks that are next used for tests
// nolint:deadcode
type conn interface {
net.Conn
}
// reader is an interface wrapper used to generate mocks that are next used for tests
// nolint:deadcode
type reader interface {
io.Reader
}
type Docker interface {
Exec(ctx context.Context, containerID string, input io.Reader, output io.Writer) error
}
func NewDocker(c docker.Client, waiter wait.KillWaiter, logger logrus.FieldLogger) Docker {
return &defaultDocker{
c: c,
waiter: waiter,
logger: logger,
}
}
type defaultDocker struct {
c docker.Client
waiter wait.KillWaiter
logger logrus.FieldLogger
}
func (d *defaultDocker) Exec(ctx context.Context, containerID string, input io.Reader, output io.Writer) error {
d.logger.Debugln("Attaching to container", containerID, "...")
hijacked, err := d.c.ContainerAttach(ctx, containerID, attachOptions())
if err != nil {
return err
}
defer hijacked.Close()
d.logger.Debugln("Starting container", containerID, "...")
err = d.c.ContainerStart(ctx, containerID, types.ContainerStartOptions{})
if err != nil {
return err
}
// Copy any output to the build trace
stdoutErrCh := make(chan error)
go func() {
_, errCopy := stdcopy.StdCopy(output, output, hijacked.Reader)
stdoutErrCh <- errCopy
}()
// Write the input to the container and close its STDIN to get it to finish
stdinErrCh := make(chan error)
go func() {
_, errCopy := io.Copy(hijacked.Conn, input)
_ = hijacked.CloseWrite()
if errCopy != nil {
stdinErrCh <- errCopy
}
}()
// Wait until either:
// - the job is aborted/cancelled/deadline exceeded
// - stdin has an error
// - stdout returns an error or nil, indicating the stream has ended and
// the container has exited
select {
case <-ctx.Done():
err = errors.New("aborted")
case err = <-stdinErrCh:
case err = <-stdoutErrCh:
}
if err != nil {
d.logger.Debugln("Container", containerID, "finished with", err)
}
// Kill and wait for exit.
// Containers are stopped so that they can be reused by the job.
return d.waiter.KillWait(ctx, containerID)
}
func attachOptions() types.ContainerAttachOptions {
return types.ContainerAttachOptions{
Stream: true,
Stdin: true,
Stdout: true,
Stderr: true,
}
}
package labels
import (
"fmt"
"strconv"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
const dockerLabelPrefix = "com.gitlab.gitlab-runner"
// Labeler is responsible for handling labelling logic for docker entities - networks, containers.
type Labeler interface {
Labels(otherLabels map[string]string) map[string]string
}
// NewLabeler returns a new instance of a Labeler bound to this build.
func NewLabeler(b *common.Build) Labeler {
return &labeler{
build: b,
}
}
type labeler struct {
build *common.Build
}
// Labels returns a map of label to value to be applied to docker entities.
// Includes a set of defaults. Add additional ones or overwrites in the provided map.
func (l *labeler) Labels(otherLabels map[string]string) map[string]string {
labels := map[string]string{
dockerLabelPrefix + ".job.id": strconv.Itoa(l.build.ID),
dockerLabelPrefix + ".job.url": l.build.JobURL(),
dockerLabelPrefix + ".job.sha": l.build.GitInfo.Sha,
dockerLabelPrefix + ".job.before_sha": l.build.GitInfo.BeforeSha,
dockerLabelPrefix + ".job.ref": l.build.GitInfo.Ref,
dockerLabelPrefix + ".project.id": strconv.Itoa(l.build.JobInfo.ProjectID),
dockerLabelPrefix + ".pipeline.id": l.build.GetAllVariables().Get("CI_PIPELINE_ID"),
dockerLabelPrefix + ".runner.id": l.build.Runner.ShortDescription(),
dockerLabelPrefix + ".runner.local_id": strconv.Itoa(l.build.RunnerID),
dockerLabelPrefix + ".managed": "true",
}
for k, v := range otherLabels {
labels[fmt.Sprintf("%s.%s", dockerLabelPrefix, k)] = v
}
return labels
}
package networks
import (
"context"
"errors"
"fmt"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/labels"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
)
var errBuildNetworkExists = errors.New("build network is not empty")
type Manager interface {
Create(ctx context.Context, networkMode string) (container.NetworkMode, error)
Inspect(ctx context.Context) (types.NetworkResource, error)
Cleanup(ctx context.Context) error
}
type manager struct {
logger debugLogger
client docker.Client
build *common.Build
labeler labels.Labeler
networkMode container.NetworkMode
buildNetwork types.NetworkResource
perBuild bool
}
func NewManager(logger debugLogger, dockerClient docker.Client, build *common.Build, labeler labels.Labeler) Manager {
return &manager{
logger: logger,
client: dockerClient,
build: build,
labeler: labeler,
}
}
func (m *manager) Create(ctx context.Context, networkMode string) (container.NetworkMode, error) {
m.networkMode = container.NetworkMode(networkMode)
m.perBuild = false
if networkMode != "" {
return m.networkMode, nil
}
if !m.build.IsFeatureFlagOn(featureflags.NetworkPerBuild) {
return m.networkMode, nil
}
if m.buildNetwork.ID != "" {
return "", errBuildNetworkExists
}
networkName := fmt.Sprintf("%s-job-%d-network", m.build.ProjectUniqueName(), m.build.ID)
m.logger.Debugln("Creating build network ", networkName)
networkResponse, err := m.client.NetworkCreate(
ctx,
networkName,
types.NetworkCreate{Labels: m.labeler.Labels(map[string]string{})},
)
if err != nil {
return "", err
}
// Inspect the created network to save its details
m.buildNetwork, err = m.client.NetworkInspect(ctx, networkResponse.ID)
if err != nil {
return "", err
}
m.networkMode = container.NetworkMode(networkName)
m.perBuild = true
return m.networkMode, nil
}
func (m *manager) Inspect(ctx context.Context) (types.NetworkResource, error) {
if !m.perBuild {
return types.NetworkResource{}, nil
}
m.logger.Debugln("Inspect docker network: ", m.buildNetwork.ID)
return m.client.NetworkInspect(ctx, m.buildNetwork.ID)
}
func (m *manager) Cleanup(ctx context.Context) error {
if !m.build.IsFeatureFlagOn(featureflags.NetworkPerBuild) {
return nil
}
if !m.perBuild {
return nil
}
m.logger.Debugln("Removing network: ", m.buildNetwork.ID)
err := m.client.NetworkRemove(ctx, m.buildNetwork.ID)
if err != nil {
return fmt.Errorf("docker remove network %s: %w", m.buildNetwork.ID, err)
}
return nil
}
package pull
import (
"context"
"fmt"
"regexp"
"strings"
"sync"
cli "github.com/docker/cli/cli/config/types"
"github.com/docker/docker/api/types"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker/auth"
)
type Manager interface {
GetDockerImage(imageName string) (*types.ImageInspect, error)
}
type ManagerConfig struct {
DockerConfig *common.DockerConfig
AuthConfig string
ShellUser string
Credentials []common.Credentials
}
type pullLogger interface {
Debugln(args ...interface{})
Infoln(args ...interface{})
Warningln(args ...interface{})
Println(args ...interface{})
}
type manager struct {
usedImages map[string]string
usedImagesLock sync.RWMutex
context context.Context
config ManagerConfig
client docker.Client
onPullImageHookFunc func()
logger pullLogger
}
func NewManager(
ctx context.Context,
logger pullLogger,
config ManagerConfig,
client docker.Client,
onPullImageHookFunc func(),
) Manager {
return &manager{
context: ctx,
client: client,
config: config,
logger: logger,
onPullImageHookFunc: onPullImageHookFunc,
}
}
func (m *manager) GetDockerImage(imageName string) (*types.ImageInspect, error) {
pullPolicies, err := m.config.DockerConfig.GetPullPolicies()
if err != nil {
return nil, err
}
var imageErr error
for idx, pullPolicy := range pullPolicies {
attempt := 1 + idx
if attempt > 1 {
m.logger.Infoln(fmt.Sprintf("Attempt #%d: Trying %q pull policy", attempt, pullPolicy))
}
var img *types.ImageInspect
img, imageErr = m.getImageUsingPullPolicy(imageName, pullPolicy)
if imageErr != nil {
m.logger.Warningln(fmt.Sprintf("Failed to pull image with policy %q: %v", pullPolicy, imageErr))
continue
}
m.markImageAsUsed(imageName, img)
return img, nil
}
return nil, fmt.Errorf(
"failed to pull image %q with specified policies %v: %w",
imageName,
pullPolicies,
imageErr,
)
}
func (m *manager) wasImageUsed(imageName, imageID string) bool {
m.usedImagesLock.RLock()
defer m.usedImagesLock.RUnlock()
return m.usedImages[imageName] == imageID
}
func (m *manager) markImageAsUsed(imageName string, image *types.ImageInspect) {
m.usedImagesLock.Lock()
defer m.usedImagesLock.Unlock()
if m.usedImages == nil {
m.usedImages = make(map[string]string)
}
m.usedImages[imageName] = image.ID
if imageName == image.ID {
return
}
if len(image.RepoDigests) > 0 {
m.logger.Println("Using docker image", image.ID, "for", imageName, "with digest", image.RepoDigests[0], "...")
} else {
m.logger.Println("Using docker image", image.ID, "for", imageName, "...")
}
}
func (m *manager) getImageUsingPullPolicy(
imageName string,
pullPolicy common.DockerPullPolicy,
) (*types.ImageInspect, error) {
m.logger.Debugln("Looking for image", imageName, "...")
existingImage, _, err := m.client.ImageInspectWithRaw(m.context, imageName)
// Return early if we already used that image
if err == nil && m.wasImageUsed(imageName, existingImage.ID) {
return &existingImage, nil
}
// If never is specified then we return what inspect did return
if pullPolicy == common.PullPolicyNever {
return &existingImage, err
}
if err == nil {
// Don't pull image that is passed by ID
if existingImage.ID == imageName {
return &existingImage, nil
}
// If not-present is specified
if pullPolicy == common.PullPolicyIfNotPresent {
m.logger.Println(fmt.Sprintf("Using locally found image version due to %q pull policy", pullPolicy))
return &existingImage, err
}
}
authConfig, err := m.resolveAuthConfigForImage(imageName)
if err != nil {
return nil, err
}
return m.pullDockerImage(imageName, authConfig)
}
func (m *manager) resolveAuthConfigForImage(imageName string) (*cli.AuthConfig, error) {
registryInfo, err := auth.ResolveConfigForImage(
imageName,
m.config.AuthConfig,
m.config.ShellUser,
m.config.Credentials,
)
if err != nil {
return nil, err
}
if registryInfo == nil {
m.logger.Debugln(fmt.Sprintf("No credentials found for %v", imageName))
return nil, nil
}
authConfig := ®istryInfo.AuthConfig
m.logger.Println(fmt.Sprintf("Authenticating with credentials from %v", registryInfo.Source))
m.logger.Debugln(fmt.Sprintf(
"Using %v to connect to %v in order to resolve %v...",
authConfig.Username,
authConfig.ServerAddress,
imageName,
))
return authConfig, nil
}
func (m *manager) pullDockerImage(imageName string, ac *cli.AuthConfig) (*types.ImageInspect, error) {
if m.onPullImageHookFunc != nil {
m.onPullImageHookFunc()
}
m.logger.Println("Pulling docker image", imageName, "...")
ref := imageName
// Add :latest to limit the download results
if !strings.ContainsAny(ref, ":@") {
ref += ":latest"
}
options := types.ImagePullOptions{}
options.RegistryAuth, _ = auth.EncodeConfig(ac)
errorRegexp := regexp.MustCompile("(repository does not exist|not found)")
if err := m.client.ImagePullBlocking(m.context, ref, options); err != nil {
if errorRegexp.MatchString(err.Error()) {
return nil, &common.BuildError{Inner: err, FailureReason: common.ScriptFailure}
}
return nil, err
}
image, _, err := m.client.ImageInspectWithRaw(m.context, imageName)
return &image, err
}
package user
import (
"bytes"
"context"
"fmt"
"strconv"
"strings"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/exec"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
)
const (
commandIDU = "id -u"
commandIDG = "id -g"
)
type Inspect interface {
IsRoot(ctx context.Context, imageID string) (bool, error)
UID(ctx context.Context, containerID string) (int, error)
GID(ctx context.Context, containerID string) (int, error)
}
func NewInspect(c docker.Client, exec exec.Docker) Inspect {
return &defaultInspect{
c: c,
exec: exec,
}
}
type defaultInspect struct {
c docker.Client
exec exec.Docker
}
func (i *defaultInspect) IsRoot(ctx context.Context, imageID string) (bool, error) {
img, _, err := i.c.ImageInspectWithRaw(ctx, imageID)
if err != nil {
return true, fmt.Errorf("inspecting container %q image: %w", imageID, err)
}
if img.Config == nil || img.Config.User == "" || img.Config.User == "root" {
return true, nil
}
return false, nil
}
func (i *defaultInspect) UID(ctx context.Context, containerID string) (int, error) {
return i.executeCommand(ctx, containerID, commandIDU)
}
func (i *defaultInspect) GID(ctx context.Context, containerID string) (int, error) {
return i.executeCommand(ctx, containerID, commandIDG)
}
func (i *defaultInspect) executeCommand(ctx context.Context, containerID string, command string) (int, error) {
input := bytes.NewBufferString(command)
output := new(bytes.Buffer)
err := i.exec.Exec(ctx, containerID, input, output)
if err != nil {
return 0, fmt.Errorf("executing %q on container %q: %w", command, containerID, err)
}
id, err := strconv.Atoi(strings.TrimSpace(output.String()))
if err != nil {
return 0, fmt.Errorf("parsing %q output: %w", command, err)
}
return id, nil
}
package volumes
import (
"context"
"errors"
"fmt"
"github.com/docker/docker/api/types/volume"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/labels"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/permission"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
)
var ErrCacheVolumesDisabled = errors.New("cache volumes feature disabled")
type Manager interface {
Create(ctx context.Context, volume string) error
CreateTemporary(ctx context.Context, destination string) error
RemoveTemporary(ctx context.Context) error
Binds() []string
}
type ManagerConfig struct {
CacheDir string
BasePath string
UniqueName string
DisableCache bool
PermissionSetter permission.Setter
}
type manager struct {
config ManagerConfig
logger debugLogger
parser parser.Parser
client docker.Client
permissionSetter permission.Setter
labeler labels.Labeler
volumeBindings []string
temporaryVolumes []string
managedVolumes pathList
}
func NewManager(
logger debugLogger,
volumeParser parser.Parser,
c docker.Client,
config ManagerConfig,
labeler labels.Labeler,
) Manager {
return &manager{
config: config,
logger: logger,
parser: volumeParser,
client: c,
volumeBindings: make([]string, 0),
managedVolumes: pathList{},
permissionSetter: config.PermissionSetter,
labeler: labeler,
}
}
// Create will create a new Docker volume bind for the specified volume. The
// volume can either be a host volume `/src:/dst`, meaning it will mount
// something from the host to the container or `/dst` which will create a Docker
// volume and mount it to the specified path.
func (m *manager) Create(ctx context.Context, volume string) error {
if len(volume) < 1 {
return nil
}
parsedVolume, err := m.parser.ParseVolume(volume)
if err != nil {
return fmt.Errorf("parse volume: %w", err)
}
switch parsedVolume.Len() {
case 2:
err = m.addHostVolume(parsedVolume)
if err != nil {
err = fmt.Errorf("adding host volume: %w", err)
}
case 1:
err = m.addCacheVolume(ctx, parsedVolume)
if err != nil {
err = fmt.Errorf("adding cache volume: %w", err)
}
default:
err = fmt.Errorf("unsupported volume definition %s", volume)
}
return err
}
func (m *manager) addHostVolume(volume *parser.Volume) error {
var err error
volume.Destination, err = m.absolutePath(volume.Destination)
if err != nil {
return fmt.Errorf("defining absolute path: %w", err)
}
err = m.managedVolumes.Add(volume.Destination)
if err != nil {
return fmt.Errorf("updating managed volume list: %w", err)
}
m.appendVolumeBind(volume)
return nil
}
func (m *manager) absolutePath(dir string) (string, error) {
if m.parser.Path().IsRoot(dir) {
return "", errDirectoryIsRootPath
}
if m.parser.Path().IsAbs(dir) {
return dir, nil
}
return m.parser.Path().Join(m.config.BasePath, dir), nil
}
func (m *manager) appendVolumeBind(volume *parser.Volume) {
m.logger.Debugln(fmt.Sprintf("Using host-based %q for %q...", volume.Source, volume.Destination))
m.volumeBindings = append(m.volumeBindings, volume.Definition())
}
func (m *manager) addCacheVolume(ctx context.Context, volume *parser.Volume) error {
// disable cache for automatic container cache,
// but leave it for host volumes (they are shared on purpose)
if m.config.DisableCache {
m.logger.Debugln("Cache containers feature is disabled")
return ErrCacheVolumesDisabled
}
if m.config.CacheDir != "" {
return m.createHostBasedCacheVolume(volume.Destination)
}
_, err := m.createCacheVolume(ctx, volume.Destination)
return err
}
func (m *manager) createHostBasedCacheVolume(destination string) error {
destination, err := m.absolutePath(destination)
if err != nil {
return fmt.Errorf("defining absolute path: %w", err)
}
err = m.managedVolumes.Add(destination)
if err != nil {
return fmt.Errorf("updating managed volumes list: %w", err)
}
hostPath := m.parser.Path().Join(m.config.CacheDir, m.config.UniqueName, hashPath(destination))
m.appendVolumeBind(&parser.Volume{
Source: hostPath,
Destination: destination,
})
return nil
}
func (m *manager) createCacheVolume(ctx context.Context, destination string) (string, error) {
destination, err := m.absolutePath(destination)
if err != nil {
return "", fmt.Errorf("defining absolute path: %w", err)
}
err = m.managedVolumes.Add(destination)
if err != nil {
return "", fmt.Errorf("updating managed volumes list: %w", err)
}
volumeName := fmt.Sprintf("%s-cache-%s", m.config.UniqueName, hashPath(destination))
vBody := volume.VolumeCreateBody{
Name: volumeName,
Labels: m.labeler.Labels(map[string]string{"type": "cache"}),
}
v, err := m.client.VolumeCreate(ctx, vBody)
if err != nil {
return "", fmt.Errorf("creating docker volume: %w", err)
}
if m.permissionSetter != nil {
err = m.permissionSetter.Set(ctx, v.Name, m.labeler.Labels(map[string]string{"type": "cache-init"}))
if err != nil {
return "", fmt.Errorf("set volume permissions: %w", err)
}
}
m.appendVolumeBind(&parser.Volume{
Source: v.Name,
Destination: destination,
})
m.logger.Debugln(fmt.Sprintf("Using volume %q as cache %q...", v.Name, destination))
return volumeName, nil
}
// CreateTemporary will create a volume, and mark it as temporary. When a volume
// is marked as temporary it means that it should be cleaned up at some point.
// It's up to the caller to clean up the temporary volumes by calling
// `RemoveTemporary`.
func (m *manager) CreateTemporary(ctx context.Context, destination string) error {
volumeName, err := m.createCacheVolume(ctx, destination)
if err != nil {
return fmt.Errorf("creating cache volume: %w", err)
}
m.temporaryVolumes = append(m.temporaryVolumes, volumeName)
return nil
}
// RemoveTemporary will remove all the volumes that are marked as temporary. If
// the volume is not found the error is ignored, any other error is returned to
// the caller.
func (m *manager) RemoveTemporary(ctx context.Context) error {
for _, v := range m.temporaryVolumes {
err := m.client.VolumeRemove(ctx, v, true)
if docker.IsErrNotFound(err) {
m.logger.Debugln(fmt.Sprintf("volume not found: %q", v))
continue
}
if err != nil {
return err
}
}
return nil
}
// Binds returns all the bindings that the volume manager is aware of.
func (m *manager) Binds() []string {
return m.volumeBindings
}
package parser
import (
"regexp"
"gitlab.com/gitlab-org/gitlab-runner/helpers/path"
)
type baseParser struct {
path path.Path
}
// The way how matchesToVolumeSpecParts parses the volume mount specification and assigns
// parts was inspired by how Docker Engine's `windowsParser` is created. The original sources
// can be found at:
//
// https://github.com/docker/engine/blob/a79fabbfe84117696a19671f4aa88b82d0f64fc1/volume/mounts/windows_parser.go
//
// The original source is licensed under Apache License 2.0 and the copyright for it
// goes to Docker, Inc.
func (p *baseParser) matchesToVolumeSpecParts(spec string, specExp *regexp.Regexp) (map[string]string, error) {
match := specExp.FindStringSubmatch(spec)
if len(match) == 0 {
return nil, NewInvalidVolumeSpecErr(spec)
}
matchgroups := make(map[string]string)
for i, name := range specExp.SubexpNames() {
matchgroups[name] = match[i]
}
parts := map[string]string{
"source": "",
"destination": "",
"mode": "",
"bindPropagation": "",
}
for group := range parts {
content, ok := matchgroups[group]
if ok {
parts[group] = content
}
}
return parts, nil
}
func (p *baseParser) Path() path.Path {
return p.path
}
package parser
import (
"fmt"
)
type InvalidVolumeSpecError struct {
spec string
}
func (e *InvalidVolumeSpecError) Error() string {
return fmt.Sprintf("invalid volume specification: %q", e.spec)
}
func NewInvalidVolumeSpecErr(spec string) error {
return &InvalidVolumeSpecError{
spec: spec,
}
}
package parser
import (
"regexp"
"gitlab.com/gitlab-org/gitlab-runner/helpers/path"
)
const (
linuxDir = `/(?:[^\\/:*?"<>|\r\n ]+/?)*`
linuxVolumeName = `[^\\/:*?"<>|\r\n]+`
linuxSource = `((?P<source>((` + linuxDir + `)|(` + linuxVolumeName + `))):)?`
linuxDestination = `(?P<destination>(?:` + linuxDir + `))`
linuxMode = `(:(?P<mode>(?i)ro|rw|z))?`
linuxBindPropagation = `((:|,)(?P<bindPropagation>(?i)shared|slave|private|rshared|rslave|rprivate))?`
)
type linuxParser struct {
baseParser
}
func NewLinuxParser() Parser {
return &linuxParser{
baseParser: baseParser{
path: path.NewUnixPath(),
},
}
}
func (p *linuxParser) ParseVolume(spec string) (*Volume, error) {
specExp := regexp.MustCompile(`^` + linuxSource + linuxDestination + linuxMode + linuxBindPropagation + `$`)
parts, err := p.matchesToVolumeSpecParts(spec, specExp)
if err != nil {
return nil, err
}
return newVolume(parts["source"], parts["destination"], parts["mode"], parts["bindPropagation"]), nil
}
package parser
import (
"strings"
)
type Volume struct {
Source string
Destination string
Mode string
BindPropagation string
}
func newVolume(source, destination string, mode string, bindPropagation string) *Volume {
return &Volume{
Source: source,
Destination: destination,
Mode: mode,
BindPropagation: bindPropagation,
}
}
func (v *Volume) Definition() string {
parts := make([]string, 0)
builder := strings.Builder{}
if v.Source != "" {
parts = append(parts, v.Source)
}
parts = append(parts, v.Destination)
if v.Mode != "" {
parts = append(parts, v.Mode)
}
builder.WriteString(strings.Join(parts, ":"))
if v.BindPropagation != "" {
separator := ":"
if v.Mode != "" {
separator = ","
}
builder.WriteString(separator)
builder.WriteString(v.BindPropagation)
}
return builder.String()
}
func (v *Volume) Len() int {
len := 0
if v.Source != "" {
len++
}
if v.Destination != "" {
len++
}
return len
}
package permission
import (
"context"
"fmt"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/wait"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
)
const dstMount = "/gitlab-runner-cache-init"
type dockerLinuxSetter struct {
client docker.Client
waiter wait.Waiter
logger logrus.FieldLogger
helperImage *types.ImageInspect
}
func NewDockerLinuxSetter(c docker.Client, logger logrus.FieldLogger, helperImage *types.ImageInspect) Setter {
return &dockerLinuxSetter{
client: c,
waiter: wait.NewDockerKillWaiter(c),
logger: logger,
helperImage: helperImage,
}
}
// Set will take the specified volume, and change the OS
// permissions so that any user can read/write to it.
//
// By default when a volume is mounted to a container it has Unix permissions
// 755, so everyone can read from it but only root can write to it. This
// prevents images that don't have root user to fail to write to mounted
// volumes.
func (d *dockerLinuxSetter) Set(ctx context.Context, volumeName string, labels map[string]string) error {
d.logger = d.logger.WithFields(logrus.Fields{
"volume_name": volumeName,
"context": "set_volume_permission",
})
containerID, err := d.createContainer(ctx, volumeName, labels)
if err != nil {
return fmt.Errorf("create permission container for volume %q: %w", volumeName, err)
}
defer func() {
removeErr := d.client.ContainerRemove(ctx, containerID, types.ContainerRemoveOptions{Force: true})
if removeErr != nil {
d.logger.WithError(removeErr).
WithField("container_id", containerID).
Debug("Failed to remove permission set container")
}
}()
err = d.runContainer(ctx, containerID)
if err != nil {
return fmt.Errorf("running permission container %q for volume %q: %w", containerID, volumeName, err)
}
return nil
}
func (d *dockerLinuxSetter) createContainer(
ctx context.Context,
volumeName string,
labels map[string]string,
) (string, error) {
volumeBinding := fmt.Sprintf("%s:%s", volumeName, dstMount)
config := &container.Config{
Image: d.helperImage.ID,
Cmd: []string{"gitlab-runner-helper", "cache-init", dstMount},
Labels: labels,
}
hostConfig := &container.HostConfig{
LogConfig: container.LogConfig{
Type: "json-file",
},
Binds: []string{volumeBinding},
}
uuid, err := helpers.GenerateRandomUUID(8)
if err != nil {
return "", fmt.Errorf("generting uuid for permission container: %v", err)
}
containerName := fmt.Sprintf("%s-set-permission-%s", volumeName, uuid)
c, err := d.client.ContainerCreate(ctx, config, hostConfig, nil, containerName)
if err != nil {
return "", err
}
d.logger.WithField("container_id", c.ID).Debug("Created container to set volume permissions")
return c.ID, err
}
func (d *dockerLinuxSetter) runContainer(ctx context.Context, containerID string) error {
err := d.client.ContainerStart(ctx, containerID, types.ContainerStartOptions{})
if err != nil {
return fmt.Errorf("starting permission container: %w", err)
}
err = d.waiter.Wait(ctx, containerID)
if err != nil {
return fmt.Errorf("waiting for permission container to finish: %w", err)
}
d.logger.WithField("container_id", containerID).Debug("Updated volume permissions")
return nil
}
package volumes
import (
"crypto/md5"
"errors"
"fmt"
"path/filepath"
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser"
)
var (
errDirectoryNotAbsolute = errors.New("build directory needs to be an absolute path")
errDirectoryIsRootPath = errors.New("build directory needs to be a non-root path")
)
type debugLogger interface {
Debugln(args ...interface{})
}
func IsHostMountedVolume(volumeParser parser.Parser, dir string, volumes ...string) (bool, error) {
if !volumeParser.Path().IsAbs(dir) {
return false, errDirectoryNotAbsolute
}
if volumeParser.Path().IsRoot(dir) {
return false, errDirectoryIsRootPath
}
for _, volume := range volumes {
parsedVolume, err := volumeParser.ParseVolume(volume)
if err != nil {
return false, err
}
if parsedVolume.Len() < 2 {
continue
}
if volumeParser.Path().Contains(parsedVolume.Destination, dir) {
return true, nil
}
}
return false, nil
}
func hashPath(path string) string {
return fmt.Sprintf("%x", md5.Sum([]byte(path)))
}
type ErrVolumeAlreadyDefined struct {
containerPath string
}
func (e *ErrVolumeAlreadyDefined) Error() string {
return fmt.Sprintf("volume for container path %q is already defined", e.containerPath)
}
func (e *ErrVolumeAlreadyDefined) Is(err error) bool {
_, ok := err.(*ErrVolumeAlreadyDefined)
return ok
}
func NewErrVolumeAlreadyDefined(containerPath string) *ErrVolumeAlreadyDefined {
return &ErrVolumeAlreadyDefined{
containerPath: containerPath,
}
}
type pathList map[string]bool
func (m pathList) Add(path string) error {
path = filepath.Clean(path)
if m[path] {
return NewErrVolumeAlreadyDefined(path)
}
m[path] = true
return nil
}
package wait
import (
"context"
"errors"
"fmt"
"time"
"github.com/docker/docker/api/types/container"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
)
type Waiter interface {
Wait(ctx context.Context, containerID string) error
}
type KillWaiter interface {
Waiter
KillWait(ctx context.Context, containerID string) error
}
type dockerWaiter struct {
client docker.Client
}
func NewDockerKillWaiter(c docker.Client) KillWaiter {
return &dockerWaiter{
client: c,
}
}
// Wait blocks until the container specified has stopped.
func (d *dockerWaiter) Wait(ctx context.Context, containerID string) error {
return d.retryWait(ctx, containerID, nil)
}
// KillWait blocks (periodically attempting to kill the container) until the
// specified container has stopped.
func (d *dockerWaiter) KillWait(ctx context.Context, containerID string) error {
return d.retryWait(ctx, containerID, func() {
_ = d.client.ContainerKill(ctx, containerID, "SIGKILL")
})
}
func (d *dockerWaiter) retryWait(ctx context.Context, containerID string, stopFn func()) error {
retries := 0
for ctx.Err() == nil {
err := d.wait(ctx, containerID, stopFn)
if err == nil {
return nil
}
var e *common.BuildError
if errors.As(err, &e) || docker.IsErrNotFound(err) || retries > 3 {
return err
}
retries++
time.Sleep(time.Second)
}
return ctx.Err()
}
// wait waits until the container has stopped.
//
// The passed `stopFn` function is periodically called (to ensure that the
// daemon absolutely receives the request) and is used to stop the container.
func (d *dockerWaiter) wait(ctx context.Context, containerID string, stopFn func()) error {
statusCh, errCh := d.client.ContainerWait(ctx, containerID, container.WaitConditionNotRunning)
if stopFn != nil {
stopFn()
}
for {
select {
case <-time.After(time.Second):
if stopFn != nil {
stopFn()
}
case err := <-errCh:
return err
case status := <-statusCh:
if status.StatusCode != 0 {
return &common.BuildError{
Inner: fmt.Errorf("exit code %d", status.StatusCode),
ExitCode: int(status.StatusCode),
}
}
return nil
}
}
}
package docker
import "gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/labels"
func (e *executor) createLabeler() error {
e.labeler = labels.NewLabeler(e.Build)
return nil
}
package machine
import (
"github.com/prometheus/client_golang/prometheus"
)
func (m *machineProvider) collectDetails() (data machinesData) {
m.lock.RLock()
defer m.lock.RUnlock()
for _, details := range m.details {
if !details.isDead() {
data.Add(details)
}
}
return
}
// Describe implements prometheus.Collector.
func (m *machineProvider) Describe(ch chan<- *prometheus.Desc) {
m.totalActions.Describe(ch)
m.creationHistogram.Describe(ch)
m.stoppingHistogram.Describe(ch)
m.removalHistogram.Describe(ch)
ch <- m.currentStatesDesc
}
// Collect implements prometheus.Collector.
func (m *machineProvider) Collect(ch chan<- prometheus.Metric) {
data := m.collectDetails()
ch <- prometheus.MustNewConstMetric(
m.currentStatesDesc,
prometheus.GaugeValue,
float64(data.Acquired),
"acquired",
)
ch <- prometheus.MustNewConstMetric(
m.currentStatesDesc,
prometheus.GaugeValue,
float64(data.Creating),
"creating",
)
ch <- prometheus.MustNewConstMetric(
m.currentStatesDesc,
prometheus.GaugeValue,
float64(data.Idle),
"idle",
)
ch <- prometheus.MustNewConstMetric(
m.currentStatesDesc,
prometheus.GaugeValue,
float64(data.Used),
"used",
)
ch <- prometheus.MustNewConstMetric(
m.currentStatesDesc,
prometheus.GaugeValue,
float64(data.Removing),
"removing",
)
ch <- prometheus.MustNewConstMetric(
m.currentStatesDesc,
prometheus.GaugeValue,
float64(data.StuckOnRemoving),
"stuck-on-removing",
)
m.totalActions.Collect(ch)
m.creationHistogram.Collect(ch)
m.stoppingHistogram.Collect(ch)
m.removalHistogram.Collect(ch)
}
package machine
import (
"sync"
)
// runnerMachinesCoordinator tracks the status of a specific Machine configuration, ensuring that the maximum number
// of concurrent machines being provisioned are limited.
type runnerMachinesCoordinator struct {
growing int
growthCondLock sync.Mutex
growthCond *sync.Cond
available uint
availableLock sync.Mutex
availableSignal chan struct{}
}
func newRunnerMachinesCoordinator() *runnerMachinesCoordinator {
coordinator := runnerMachinesCoordinator{}
coordinator.availableSignal = make(chan struct{})
coordinator.growthCond = sync.NewCond(&coordinator.growthCondLock)
return &coordinator
}
func (r *runnerMachinesCoordinator) waitForGrowthCapacity(maxGrowth int, f func()) {
r.growthCondLock.Lock()
for maxGrowth != 0 && r.growing >= maxGrowth {
r.growthCond.Wait()
}
r.growing++
r.growthCondLock.Unlock()
defer func() {
r.growthCondLock.Lock()
r.growing--
r.growthCondLock.Unlock()
r.growthCond.Signal()
}()
f()
}
// getAvailableMachine returns whether there is a machine available.
// It reduces the internal counter if it can be reduced so next time it might return
// a different value.
func (r *runnerMachinesCoordinator) getAvailableMachine() bool {
r.availableLock.Lock()
defer r.availableLock.Unlock()
if r.available == 0 {
return false
}
r.available--
return true
}
// addAvailableMachine increments an internal counter which
// is used by getAvailableMachine to check for availability.
func (r *runnerMachinesCoordinator) addAvailableMachine() {
r.availableLock.Lock()
defer r.availableLock.Unlock()
r.available++
select {
case r.availableSignal <- struct{}{}:
default:
}
}
func (r *runnerMachinesCoordinator) availableMachineSignal() <-chan struct{} {
return r.availableSignal
}
type runnersDetails map[string]*runnerMachinesCoordinator
package machine
import (
"fmt"
"os"
"time"
"github.com/sirupsen/logrus"
)
type machinesData struct {
Runner string
Acquired int
Creating int
Idle int
Used int
Removing int
StuckOnRemoving int
}
func (d *machinesData) Available() int {
return d.Acquired + d.Creating + d.Idle
}
func (d *machinesData) Total() int {
return d.Acquired + d.Creating + d.Idle + d.Used + d.Removing + d.StuckOnRemoving
}
func (d *machinesData) Add(details *machineDetails) {
switch details.State {
case machineStateIdle:
d.Idle++
case machineStateCreating:
d.Creating++
case machineStateAcquired:
d.Acquired++
case machineStateUsed:
d.Used++
case machineStateRemoving:
if details.isStuckOnRemove() {
d.StuckOnRemoving++
} else {
d.Removing++
}
}
}
func (d *machinesData) Fields() logrus.Fields {
return logrus.Fields{
"runner": d.Runner,
"used": d.Used,
"idle": d.Idle,
"total": d.Total(),
"creating": d.Creating,
"removing": d.Removing,
}
}
func (d *machinesData) writeDebugInformation() {
if logrus.GetLevel() < logrus.DebugLevel {
return
}
file, err := os.OpenFile("machines.csv", os.O_RDWR|os.O_APPEND, 0600)
if err != nil {
return
}
defer func() { _ = file.Close() }()
_, _ = fmt.Fprintln(
file,
"time", time.Now(),
"runner", d.Runner,
"acquired", d.Acquired,
"creating", d.Creating,
"idle", d.Idle,
"used", d.Used,
"removing", d.Removing,
)
}
package machine
import (
"fmt"
"io/ioutil"
"time"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
)
type machineDetails struct {
Name string
Created time.Time `yaml:"-"`
Used time.Time `yaml:"-"`
UsedCount int
State machineState
Reason string
RetryCount int
LastSeen time.Time
}
func (m *machineDetails) isPersistedOnDisk() bool {
// Machines in creating phase might or might not be persisted on disk
// this is due to async nature of machine creation process
// where to `docker-machine create` is the one that is creating relevant files
// and it is being executed with undefined delay
return m.State != machineStateCreating
}
func (m *machineDetails) isUsed() bool {
return m.State != machineStateIdle
}
func (m *machineDetails) isStuckOnRemove() bool {
return m.State == machineStateRemoving && m.RetryCount >= removeRetryTries
}
func (m *machineDetails) isDead() bool {
return m.State == machineStateIdle &&
time.Since(m.LastSeen) > machineDeadInterval
}
func (m *machineDetails) canBeUsed() bool {
return m.State == machineStateAcquired
}
func (m *machineDetails) match(machineFilter string) bool {
var query string
if n, _ := fmt.Sscanf(m.Name, machineFilter, &query); n != 1 {
return false
}
return true
}
func (m *machineDetails) writeDebugInformation() {
if logrus.GetLevel() < logrus.DebugLevel {
return
}
var details struct {
Details machineDetails
Time string
CreatedAgo time.Duration
}
details.Details = *m
details.Time = time.Now().String()
details.CreatedAgo = time.Since(m.Created)
data := helpers.ToYAML(&details)
_ = ioutil.WriteFile("machines/"+details.Details.Name+".yml", []byte(data), 0600)
}
func (m *machineDetails) logger() *logrus.Entry {
return logrus.WithFields(logrus.Fields{
"name": m.Name,
"lifetime": time.Since(m.Created),
"used": time.Since(m.Used),
"usedCount": m.UsedCount,
"reason": m.Reason,
})
}
type machinesDetails map[string]*machineDetails
package machine
import (
"errors"
"time"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
_ "gitlab.com/gitlab-org/gitlab-runner/executors/docker" // Force to load docker executor
"gitlab.com/gitlab-org/gitlab-runner/referees"
)
const (
DockerMachineExecutorStageUseMachine common.ExecutorStage = "docker_machine_use_machine"
DockerMachineExecutorStageReleaseMachine common.ExecutorStage = "docker_machine_release_machine"
)
type machineExecutor struct {
provider *machineProvider
executor common.Executor
build *common.Build
data common.ExecutorData
config common.RunnerConfig
currentStage common.ExecutorStage
}
func (e *machineExecutor) log() (log *logrus.Entry) {
log = e.build.Log()
details, _ := e.build.ExecutorData.(*machineDetails)
if details == nil {
details, _ = e.data.(*machineDetails)
}
if details != nil {
log = log.WithFields(logrus.Fields{
"name": details.Name,
"usedcount": details.UsedCount,
"created": details.Created,
"now": time.Now(),
})
}
if e.config.Docker != nil {
log = log.WithField("docker", e.config.Docker.Host)
}
return
}
func (e *machineExecutor) Shell() *common.ShellScriptInfo {
if e.executor == nil {
return nil
}
return e.executor.Shell()
}
func (e *machineExecutor) Prepare(options common.ExecutorPrepareOptions) (err error) {
e.build = options.Build
if options.Config.Docker == nil {
options.Config.Docker = &common.DockerConfig{}
}
// Use the machine
e.SetCurrentStage(DockerMachineExecutorStageUseMachine)
e.config, e.data, err = e.provider.Use(options.Config, options.Build.ExecutorData)
if err != nil {
return err
}
options.Config.Docker.Credentials = e.config.Docker.Credentials
// TODO: Currently the docker-machine doesn't support multiple builds
e.build.ProjectRunnerID = 0
if details, _ := options.Build.ExecutorData.(*machineDetails); details != nil {
options.Build.Hostname = details.Name
} else if details, _ := e.data.(*machineDetails); details != nil {
options.Build.Hostname = details.Name
}
// e.data is only set if the docker-machine created is new
if e.data == nil {
e.log().Infoln("Using existing docker-machine")
} else {
e.log().Infoln("Created docker-machine")
}
// Create original executor
e.executor = e.provider.provider.Create()
if e.executor == nil {
return errors.New("failed to create an executor")
}
if err = e.executor.Prepare(options); err != nil {
e.log().Infoln("Preparing docker-machine wrapped executor failed")
return err
}
e.log().Infoln("Starting docker-machine build...")
return nil
}
func (e *machineExecutor) Run(cmd common.ExecutorCommand) error {
if e.executor == nil {
return errors.New("missing executor")
}
return e.executor.Run(cmd)
}
func (e *machineExecutor) Finish(err error) {
if e.executor != nil {
e.executor.Finish(err)
}
e.log().Infoln("Finished docker-machine build:", err)
}
func (e *machineExecutor) Cleanup() {
// Cleanup executor if were created
if e.executor != nil {
e.executor.Cleanup()
}
e.log().Infoln("Cleaned up docker-machine")
// Release allocated machine
if e.data != nil {
e.SetCurrentStage(DockerMachineExecutorStageReleaseMachine)
e.provider.Release(&e.config, e.data)
e.data = nil
}
}
func (e *machineExecutor) GetCurrentStage() common.ExecutorStage {
if e.executor == nil {
return common.ExecutorStage("")
}
return e.executor.GetCurrentStage()
}
func (e *machineExecutor) SetCurrentStage(stage common.ExecutorStage) {
if e.executor == nil {
e.currentStage = stage
return
}
e.executor.SetCurrentStage(stage)
}
func (e *machineExecutor) GetMetricsSelector() string {
refereed, ok := e.executor.(referees.MetricsExecutor)
if !ok {
return ""
}
return refereed.GetMetricsSelector()
}
func init() {
common.RegisterExecutorProvider("docker+machine", newMachineProvider("docker+machine", "docker"))
common.RegisterExecutorProvider("docker-ssh+machine", newMachineProvider("docker-ssh+machine", "docker-ssh"))
}
package machine
import (
"crypto/rand"
"fmt"
"strings"
"time"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/dns"
)
func machineFormat(runner string, template string) string {
if runner != "" {
return "runner-" + strings.ToLower(runner) + "-" + template
}
return template
}
func machineFilter(config *common.RunnerConfig) string {
return machineFormat(dns.MakeRFC1123Compatible(config.ShortDescription()), config.Machine.MachineName)
}
func matchesMachineFilter(name, filter string) bool {
var query string
if n, _ := fmt.Sscanf(name, filter, &query); n == 1 {
return true
}
return false
}
func filterMachineList(machines []string, filter string) (newMachines []string) {
newMachines = make([]string, 0, len(machines))
for _, machine := range machines {
if matchesMachineFilter(machine, filter) {
newMachines = append(newMachines, machine)
}
}
return
}
func newMachineName(config *common.RunnerConfig) string {
r := make([]byte, 4)
_, _ = rand.Read(r)
t := time.Now().Unix()
return fmt.Sprintf(machineFilter(config), fmt.Sprintf("%d-%x", t, r))
}
package machine
import (
"errors"
"fmt"
"sync"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
)
type machineProvider struct {
name string
machine docker.Machine
details machinesDetails
runners runnersDetails
lock sync.RWMutex
acquireLock sync.Mutex
// provider stores a real executor that is used to start run the builds
provider common.ExecutorProvider
stuckRemoveLock sync.Mutex
// metrics
totalActions *prometheus.CounterVec
currentStatesDesc *prometheus.Desc
creationHistogram prometheus.Histogram
stoppingHistogram prometheus.Histogram
removalHistogram prometheus.Histogram
}
func (m *machineProvider) machineDetails(name string, acquire bool) *machineDetails {
details := m.ensureDetails(name)
if acquire {
details = m.tryAcquireMachineDetails(details)
}
return details
}
func (m *machineProvider) ensureDetails(name string) *machineDetails {
m.lock.Lock()
defer m.lock.Unlock()
details, ok := m.details[name]
if !ok {
now := time.Now()
details = &machineDetails{
Name: name,
Created: now,
Used: now,
LastSeen: now,
UsedCount: 1, // any machine that we find we mark as already used
State: machineStateIdle,
}
m.details[name] = details
}
return details
}
var errNoConfig = errors.New("no runner config specified")
func (m *machineProvider) runnerMachinesCoordinator(config *common.RunnerConfig) (*runnerMachinesCoordinator, error) {
if config == nil {
return nil, errNoConfig
}
m.lock.Lock()
defer m.lock.Unlock()
details, ok := m.runners[config.GetToken()]
if !ok {
details = newRunnerMachinesCoordinator()
m.runners[config.GetToken()] = details
}
return details, nil
}
func (m *machineProvider) create(config *common.RunnerConfig, state machineState) (*machineDetails, chan error) {
name := newMachineName(config)
details := m.machineDetails(name, true)
m.lock.Lock()
details.State = machineStateCreating
details.UsedCount = 0
details.RetryCount = 0
details.LastSeen = time.Now()
m.lock.Unlock()
errCh := make(chan error, 1)
// Create machine with the required configuration asynchronously
coordinator, err := m.runnerMachinesCoordinator(config)
if err != nil {
errCh <- err
return nil, errCh
}
go coordinator.waitForGrowthCapacity(config.Machine.MaxGrowthRate, func() {
m.createWithGrowthCapacity(coordinator, config, details, state, errCh)
})
return details, errCh
}
func (m *machineProvider) createWithGrowthCapacity(
coordinator *runnerMachinesCoordinator,
config *common.RunnerConfig,
details *machineDetails,
state machineState,
errCh chan error,
) {
logger := logrus.WithField("name", details.Name)
started := time.Now()
err := m.reprovisionMachineOnCreationFailure(
logger,
config,
details,
m.machine.Create(config.Machine.MachineDriver, details.Name, config.Machine.MachineOptions...),
)
if err != nil {
logger.WithField("time", time.Since(started)).
WithError(err).
Errorln("Machine creation failed")
_ = m.remove(details.Name, "Failed to create")
} else {
m.lock.Lock()
details.State = state
details.Used = time.Now()
m.lock.Unlock()
creationTime := time.Since(started)
m.lock.RLock()
logger.WithField("duration", creationTime).
WithField("now", time.Now()).
WithField("retries", details.RetryCount).
Infoln("Machine created")
m.lock.RUnlock()
m.totalActions.WithLabelValues("created").Inc()
m.creationHistogram.Observe(creationTime.Seconds())
// Signal that a new machine is available. When there's contention, there's no guarantee between the
// ordering of reading from errCh and the availability check.
coordinator.addAvailableMachine()
}
errCh <- err
}
func (m *machineProvider) reprovisionMachineOnCreationFailure(
logger logrus.FieldLogger,
config *common.RunnerConfig,
details *machineDetails,
err error,
) error {
skipProvision := config.IsFeatureFlagOn(featureflags.SkipDockerMachineProvisionOnCreationFailure)
if skipProvision {
logger.WithError(err).Infof("Skipping provision retry on failed machine")
return err
}
for i := 0; i < 3 && err != nil; i++ {
details.RetryCount++
logger.WithError(err).
Warningln("Machine creation failed, trying to provision")
time.Sleep(provisionRetryInterval)
err = m.machine.Provision(details.Name)
}
return err
}
func (m *machineProvider) findFreeMachine(skipCache bool, machines ...string) (details *machineDetails) {
// Enumerate all machines in reverse order, to always take the newest machines first
for idx := range machines {
name := machines[len(machines)-idx-1]
details := m.machineDetails(name, true)
if details == nil {
continue
}
// Check if node is running
canConnect := m.machine.CanConnect(name, skipCache)
if !canConnect {
_ = m.remove(name, "machine is unavailable")
continue
}
return details
}
return nil
}
func (m *machineProvider) findFreeExistingMachine(config *common.RunnerConfig) (*machineDetails, error) {
machines, err := m.loadMachines(config)
if err != nil {
return nil, err
}
return m.findFreeMachine(true, machines...), nil
}
func (m *machineProvider) useMachine(config *common.RunnerConfig) (*machineDetails, error) {
details, err := m.findFreeExistingMachine(config)
if err != nil || details != nil {
return details, err
}
return m.createAndAcquireMachine(config)
}
func (m *machineProvider) createAndAcquireMachine(config *common.RunnerConfig) (*machineDetails, error) {
coordinator, err := m.runnerMachinesCoordinator(config)
if err != nil {
return nil, err
}
newDetails, errCh := m.create(config, machineStateIdle)
// Use either a free machine, or the created machine; whichever comes first. There's no guarantee that the created
// machine can be used by us because between the time the machine is created, and the acquisition of the machine,
// another goroutine may have found it via findFreeMachine and acquired it.
var details *machineDetails
for details == nil && err == nil {
select {
case err = <-errCh:
if err != nil {
return nil, err
}
details = m.tryAcquireMachineDetails(newDetails)
case <-coordinator.availableMachineSignal():
// Even though the signal is fired and we are *almost* sure that
// there's a machine available, let's use the getAvailableMachine
// method so that the internal counter is synchonized with what
// we are actually doing and so that we can be sure that no other
// goroutine that didn't accept the signal and instead used the ticker
// hasn't already snatched a machine
details, err = m.tryGetFreeExistingMachineFromCoordinator(config, coordinator)
case <-time.After(time.Second):
details, err = m.tryGetFreeExistingMachineFromCoordinator(config, coordinator)
}
}
return details, err
}
func (m *machineProvider) tryGetFreeExistingMachineFromCoordinator(
config *common.RunnerConfig,
coordinator *runnerMachinesCoordinator,
) (*machineDetails, error) {
if coordinator.getAvailableMachine() {
return m.findFreeExistingMachine(config)
}
return nil, nil
}
func (m *machineProvider) tryAcquireMachineDetails(details *machineDetails) *machineDetails {
m.lock.Lock()
defer m.lock.Unlock()
if details.isUsed() {
return nil
}
details.State = machineStateAcquired
return details
}
func (m *machineProvider) retryUseMachine(config *common.RunnerConfig) (details *machineDetails, err error) {
// Try to find a machine
for i := 0; i < 3; i++ {
details, err = m.useMachine(config)
if err == nil {
break
}
time.Sleep(provisionRetryInterval)
}
return
}
func (m *machineProvider) removeMachine(details *machineDetails) (err error) {
if !m.machine.Exist(details.Name) {
details.logger().
Warningln("Skipping machine removal, because it doesn't exist")
return nil
}
// This code limits amount of removal of stuck machines to one machine per interval
if details.isStuckOnRemove() {
m.stuckRemoveLock.Lock()
defer m.stuckRemoveLock.Unlock()
}
details.logger().Warningln("Stopping machine")
err = runHistogramCountedOperation(m.stoppingHistogram, func() error {
return m.machine.Stop(details.Name, machineStopCommandTimeout)
})
if err != nil {
details.logger().
WithError(err).
Warningln("Error while stopping machine")
}
details.logger().Warningln("Removing machine")
err = runHistogramCountedOperation(m.removalHistogram, func() error {
return m.machine.Remove(details.Name)
})
if err != nil {
details.RetryCount++
time.Sleep(removeRetryInterval)
return err
}
return nil
}
func runHistogramCountedOperation(histogram prometheus.Histogram, operation func() error) error {
startedAt := time.Now()
err := operation()
histogram.Observe(time.Since(startedAt).Seconds())
return err
}
func (m *machineProvider) finalizeRemoval(details *machineDetails) {
for {
err := m.removeMachine(details)
if err == nil {
break
}
}
m.lock.Lock()
defer m.lock.Unlock()
delete(m.details, details.Name)
details.logger().
WithField("now", time.Now()).
WithField("retries", details.RetryCount).
Infoln("Machine removed")
m.totalActions.WithLabelValues("removed").Inc()
}
func (m *machineProvider) remove(machineName string, reason ...interface{}) error {
m.lock.Lock()
defer m.lock.Unlock()
details := m.details[machineName]
if details == nil {
return errors.New("machine not found")
}
details.Reason = fmt.Sprint(reason...)
details.State = machineStateRemoving
details.RetryCount = 0
details.logger().
WithField("now", time.Now()).
Warningln("Requesting machine removal")
details.Used = time.Now()
details.writeDebugInformation()
go m.finalizeRemoval(details)
return nil
}
func (m *machineProvider) updateMachine(
config *common.RunnerConfig,
data *machinesData,
details *machineDetails,
) error {
if details.State != machineStateIdle {
return nil
}
if config.Machine.MaxBuilds > 0 && details.UsedCount >= config.Machine.MaxBuilds {
// Limit number of builds
return errors.New("too many builds")
}
if data.Total() >= config.Limit && config.Limit > 0 {
// Limit maximum number of machines
return errors.New("too many machines")
}
if time.Since(details.Used) > time.Second*time.Duration(config.Machine.GetIdleTime()) {
if data.Idle >= config.Machine.GetIdleCount() {
// Remove machine that are way over the idle time
return errors.New("too many idle machines")
}
}
return nil
}
func (m *machineProvider) updateMachines(
machines []string,
config *common.RunnerConfig,
) (data machinesData, validMachines []string) {
data.Runner = config.ShortDescription()
validMachines = make([]string, 0, len(machines))
for _, name := range machines {
details := m.machineDetails(name, false)
details.LastSeen = time.Now()
err := m.updateMachine(config, &data, details)
if err == nil {
validMachines = append(validMachines, name)
} else {
_ = m.remove(details.Name, err)
}
data.Add(details)
}
return
}
func (m *machineProvider) createMachines(config *common.RunnerConfig, data *machinesData) {
// Create a new machines and mark them as Idle
for {
if data.Available() >= config.Machine.GetIdleCount() {
// Limit maximum number of idle machines
break
}
if data.Total() >= config.Limit && config.Limit > 0 {
// Limit maximum number of machines
break
}
if data.Creating >= config.Machine.MaxGrowthRate && config.Machine.MaxGrowthRate > 0 {
// Prevent excessive growth in the number of machines
break
}
m.create(config, machineStateIdle)
data.Creating++
}
}
// intermediateMachineList returns a list of machines that might not yet be
// persisted on disk, these machines are the ones between being virtually
// created, and `docker-machine create` getting executed we populate this data
// set to overcome the race conditions related to not-full set of machines
// returned by `docker-machine ls -q`
func (m *machineProvider) intermediateMachineList(excludedMachines []string) []string {
var excludedSet map[string]struct{}
var intermediateMachines []string
m.lock.Lock()
defer m.lock.Unlock()
for _, details := range m.details {
if details.isPersistedOnDisk() {
continue
}
// lazy init set, as most of times we don't create new machines
if excludedSet == nil {
excludedSet = make(map[string]struct{}, len(excludedMachines))
for _, excludedMachine := range excludedMachines {
excludedSet[excludedMachine] = struct{}{}
}
}
if _, ok := excludedSet[details.Name]; ok {
continue
}
intermediateMachines = append(intermediateMachines, details.Name)
}
return intermediateMachines
}
func (m *machineProvider) loadMachines(config *common.RunnerConfig) (machines []string, err error) {
machines, err = m.machine.List()
if err != nil {
return nil, err
}
machines = append(machines, m.intermediateMachineList(machines)...)
machines = filterMachineList(machines, machineFilter(config))
return
}
func (m *machineProvider) Acquire(config *common.RunnerConfig) (common.ExecutorData, error) {
if config.Machine == nil || config.Machine.MachineName == "" {
return nil, fmt.Errorf("missing Machine options")
}
// Lock updating machines, because two Acquires can be run at the same time
m.acquireLock.Lock()
defer m.acquireLock.Unlock()
machines, err := m.loadMachines(config)
if err != nil {
return nil, err
}
// Update a list of currently configured machines
machinesData, validMachines := m.updateMachines(machines, config)
// Pre-create machines
m.createMachines(config, &machinesData)
logrus.WithFields(machinesData.Fields()).
WithField("runner", config.ShortDescription()).
WithField("minIdleCount", config.Machine.GetIdleCount()).
WithField("maxMachines", config.Limit).
WithField("maxMachineCreate", config.Machine.MaxGrowthRate).
WithField("time", time.Now()).
Debugln("Docker Machine Details")
machinesData.writeDebugInformation()
// Try to find a free machine
details := m.findFreeMachine(false, validMachines...)
if details != nil {
return details, nil
}
// If we have a free machines we can process a build
if config.Machine.GetIdleCount() != 0 && machinesData.Idle == 0 {
err = errors.New("no free machines that can process builds")
}
return nil, err
}
//nolint:nakedret
func (m *machineProvider) Use(
config *common.RunnerConfig,
data common.ExecutorData,
) (newConfig common.RunnerConfig, newData common.ExecutorData, err error) {
// Find a new machine
details, _ := data.(*machineDetails)
if details == nil || !details.canBeUsed() || !m.machine.CanConnect(details.Name, true) {
details, err = m.retryUseMachine(config)
if err != nil {
return
}
// Return details only if this is a new instance
newData = details
}
// Get machine credentials
dc, err := m.machine.Credentials(details.Name)
if err != nil {
if newData != nil {
m.Release(config, newData)
}
newData = nil
return
}
// Create shallow copy of config and store in it docker credentials
newConfig = *config
newConfig.Docker = &common.DockerConfig{}
if config.Docker != nil {
*newConfig.Docker = *config.Docker
}
newConfig.Docker.Credentials = dc
// Mark machine as used
details.State = machineStateUsed
details.Used = time.Now()
details.UsedCount++
m.totalActions.WithLabelValues("used").Inc()
return
}
func (m *machineProvider) Release(config *common.RunnerConfig, data common.ExecutorData) {
// Release machine
details, ok := data.(*machineDetails)
if !ok {
return
}
m.lock.Lock()
// Mark last used time when is Used
if details.State == machineStateUsed {
details.Used = time.Now()
}
m.lock.Unlock()
// Remove machine if we already used it
if config != nil && config.Machine != nil &&
config.Machine.MaxBuilds > 0 && details.UsedCount >= config.Machine.MaxBuilds {
err := m.remove(details.Name, "Too many builds")
if err == nil {
return
}
}
m.lock.Lock()
details.State = machineStateIdle
m.lock.Unlock()
// Signal pending builds that a new machine is available.
if err := m.signalRelease(config); err != nil {
return
}
}
func (m *machineProvider) signalRelease(config *common.RunnerConfig) error {
coordinator, err := m.runnerMachinesCoordinator(config)
if err != nil && err != errNoConfig {
return err
}
if err != errNoConfig && coordinator != nil {
coordinator.addAvailableMachine()
}
return nil
}
func (m *machineProvider) CanCreate() bool {
return m.provider.CanCreate()
}
func (m *machineProvider) GetFeatures(features *common.FeaturesInfo) error {
return m.provider.GetFeatures(features)
}
func (m *machineProvider) GetConfigInfo(input *common.RunnerConfig, output *common.ConfigInfo) {
m.provider.GetConfigInfo(input, output)
}
func (m *machineProvider) GetDefaultShell() string {
return m.provider.GetDefaultShell()
}
func (m *machineProvider) Create() common.Executor {
return &machineExecutor{
provider: m,
}
}
func newMachineProvider(name, executor string) *machineProvider {
provider := common.GetExecutorProvider(executor)
if provider == nil {
logrus.Panicln("Missing", executor)
}
return &machineProvider{
name: name,
details: make(machinesDetails),
runners: make(runnersDetails),
machine: docker.NewMachineCommand(),
provider: provider,
totalActions: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "gitlab_runner_autoscaling_actions_total",
Help: "The total number of actions executed by the provider.",
ConstLabels: prometheus.Labels{
"executor": name,
},
},
[]string{"action"},
),
currentStatesDesc: prometheus.NewDesc(
"gitlab_runner_autoscaling_machine_states",
"The current number of machines per state in this provider.",
[]string{"state"},
prometheus.Labels{
"executor": name,
},
),
creationHistogram: prometheus.NewHistogram(
prometheus.HistogramOpts{
Name: "gitlab_runner_autoscaling_machine_creation_duration_seconds",
Help: "Histogram of machine creation time.",
Buckets: prometheus.ExponentialBuckets(30, 1.25, 10),
ConstLabels: prometheus.Labels{
"executor": name,
},
},
),
stoppingHistogram: prometheus.NewHistogram(
prometheus.HistogramOpts{
Name: "gitlab_runner_autoscaling_machine_stopping_duration_seconds",
Help: "Histogram of machine stopping time.",
Buckets: []float64{1, 3, 5, 10, 30, 50, 60, 80, 90, 120},
ConstLabels: prometheus.Labels{
"executor": name,
},
},
),
removalHistogram: prometheus.NewHistogram(
prometheus.HistogramOpts{
Name: "gitlab_runner_autoscaling_machine_removal_duration_seconds",
Help: "Histogram of machine removal time.",
Buckets: []float64{1, 3, 5, 10, 30, 50, 60, 80, 90, 120},
ConstLabels: prometheus.Labels{
"executor": name,
},
},
),
}
}
package machine
type machineState int
const (
machineStateIdle machineState = iota
machineStateAcquired
machineStateCreating
machineStateUsed
machineStateRemoving
)
func (t machineState) String() string {
switch t {
case machineStateIdle:
return "Idle"
case machineStateAcquired:
return "Acquired"
case machineStateCreating:
return "Creating"
case machineStateUsed:
return "Used"
case machineStateRemoving:
return "Removing"
default:
return "Unknown"
}
}
func (t machineState) MarshalText() ([]byte, error) {
return []byte(t.String()), nil
}
package machine
import (
"errors"
"gitlab.com/gitlab-org/gitlab-runner/session/terminal"
)
func (e *machineExecutor) Connect() (terminal.Conn, error) {
if term, ok := e.executor.(terminal.InteractiveTerminal); ok {
return term.Connect()
}
return nil, errors.New("executor does not have terminal")
}
package docker
import (
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/networks"
)
var createNetworksManager = func(e *executor) (networks.Manager, error) {
networksManager := networks.NewManager(&e.BuildLogger, e.client, e.Build, e.labeler)
return networksManager, nil
}
func (e *executor) createNetworksManager() error {
nm, err := createNetworksManager(e)
if err != nil {
return err
}
e.networksManager = nm
return nil
}
package docker
import (
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/pull"
)
var createPullManager = func(e *executor) (pull.Manager, error) {
config := pull.ManagerConfig{
DockerConfig: e.Config.Docker,
AuthConfig: e.Build.GetDockerAuthConfig(),
ShellUser: e.Shell().User,
Credentials: e.Build.Credentials,
}
pullManager := pull.NewManager(e.Context, &e.BuildLogger, config, e.client, func() {
e.SetCurrentStage(ExecutorStagePullingImage)
})
return pullManager, nil
}
func (e *executor) createPullManager() error {
pm, err := createPullManager(e)
if err != nil {
return err
}
e.pullManager = pm
return nil
}
package docker
import (
"context"
"errors"
"fmt"
"net/http"
"time"
"github.com/docker/docker/api/types"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
terminalsession "gitlab.com/gitlab-org/gitlab-runner/session/terminal"
terminal "gitlab.com/gitlab-org/gitlab-terminal"
)
// buildContainerTerminalTimeout is the error used when the build container is
// not running yet an we have a terminal request waiting for the container to
// start and a certain amount of time is exceeded.
type buildContainerTerminalTimeout struct {
}
func (buildContainerTerminalTimeout) Error() string {
return "timeout for waiting for build container"
}
func (s *commandExecutor) watchForRunningBuildContainer(deadline time.Time) (string, error) {
for time.Since(deadline) < 0 {
buildContainer := s.getBuildContainer()
if buildContainer == nil {
time.Sleep(time.Second)
continue
}
containerID := buildContainer.ID
container, err := s.client.ContainerInspect(s.Context, containerID)
if err != nil {
return "", err
}
if container.State.Running {
return containerID, nil
}
}
return "", buildContainerTerminalTimeout{}
}
func (s *commandExecutor) Connect() (terminalsession.Conn, error) {
// Waiting for the container to start, is not ideal as it might be hiding a
// real issue and the user is not aware of it. Ideally, the runner should
// inform the user in an interactive way that the container has no started
// yet and should wait/try again. This isn't an easy task to do since we
// can't access the WebSocket here since that is the responsibility of
// `gitlab-terminal` package. There are plans to improve this please take a
// look at https://gitlab.com/gitlab-org/gitlab-ce/issues/50384#proposal and
// https://gitlab.com/gitlab-org/gitlab-terminal/issues/4
timeout := s.terminalWaitForContainerTimeout
if timeout == 0 {
timeout = waitForContainerTimeout
}
containerID, err := s.watchForRunningBuildContainer(time.Now().Add(timeout))
if err != nil {
return nil, err
}
ctx, cancelFn := context.WithCancel(s.Context)
return terminalConn{
logger: &s.BuildLogger,
ctx: ctx,
cancelFn: cancelFn,
executor: s,
client: s.client,
containerID: containerID,
shell: s.BuildShell.DockerCommand,
}, nil
}
type terminalConn struct {
logger *common.BuildLogger
ctx context.Context
cancelFn func()
executor *commandExecutor
client docker.Client
containerID string
shell []string
}
func (t terminalConn) Start(w http.ResponseWriter, r *http.Request, timeoutCh, disconnectCh chan error) {
execConfig := types.ExecConfig{
Tty: true,
AttachStdin: true,
AttachStderr: true,
AttachStdout: true,
Cmd: t.shell,
}
exec, err := t.client.ContainerExecCreate(t.ctx, t.containerID, execConfig)
if err != nil {
t.logger.Errorln("Failed to create exec container for terminal:", err)
http.Error(w, "failed to create exec for build container", http.StatusInternalServerError)
return
}
execStartCfg := types.ExecStartCheck{Tty: true}
resp, err := t.client.ContainerExecAttach(t.ctx, exec.ID, execStartCfg)
if err != nil {
t.logger.Errorln("Failed to exec attach to container for terminal:", err)
http.Error(w, "failed to attach tty to build container", http.StatusInternalServerError)
return
}
dockerTTY := newDockerTTY(&resp)
proxy := terminal.NewStreamProxy(1) // one stopper: terminal exit handler
// wait for container to exit
go func() {
t.logger.Debugln("Waiting for the terminal container:", t.containerID)
err := t.executor.waiter.Wait(t.ctx, t.containerID)
t.logger.Debugln("The terminal container:", t.containerID, "finished with:", err)
stopCh := proxy.GetStopCh()
if err != nil {
stopCh <- fmt.Errorf("build container exited with %w", err)
} else {
stopCh <- errors.New("build container exited")
}
}()
terminalsession.ProxyTerminal(
timeoutCh,
disconnectCh,
proxy.StopCh,
func() {
terminal.ProxyStream(w, r, dockerTTY, proxy)
},
)
}
func (t terminalConn) Close() error {
if t.cancelFn != nil {
t.cancelFn()
}
return nil
}
package docker
import "github.com/docker/docker/api/types"
func newDockerTTY(hijackedResp *types.HijackedResponse) *dockerTTY {
return &dockerTTY{
hijackedResp: hijackedResp,
}
}
type dockerTTY struct {
hijackedResp *types.HijackedResponse
}
func (d *dockerTTY) Read(p []byte) (int, error) {
return d.hijackedResp.Reader.Read(p)
}
func (d *dockerTTY) Write(p []byte) (int, error) {
return d.hijackedResp.Conn.Write(p)
}
func (d *dockerTTY) Close() error {
d.hijackedResp.Close()
_ = d.hijackedResp.CloseWrite()
return nil
}
package docker
import (
"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes"
)
var createVolumesManager = func(e *executor) (volumes.Manager, error) {
config := volumes.ManagerConfig{
CacheDir: e.Config.Docker.CacheDir,
BasePath: e.Build.FullProjectDir(),
UniqueName: e.Build.ProjectUniqueName(),
DisableCache: e.Config.Docker.DisableCache,
}
if e.newVolumePermissionSetter != nil {
setter, err := e.newVolumePermissionSetter()
if err != nil {
return nil, err
}
config.PermissionSetter = setter
}
volumesManager := volumes.NewManager(&e.BuildLogger, e.volumeParser, e.client, config, e.labeler)
return volumesManager, nil
}
func (e *executor) createVolumesManager() error {
vm, err := createVolumesManager(e)
if err != nil {
return err
}
e.volumesManager = vm
return nil
}
/*
Copyright 2014 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This file was modified by James Munnelly (https://gitlab.com/u/munnerz)
*/
package kubernetes
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"github.com/sirupsen/logrus"
api "k8s.io/api/core/v1"
kubeerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/remotecommand"
)
const (
commandConnectFailureMaxTries = 5
errorDialingBackendEOFMessage = "error dialing backend: EOF"
)
// RemoteExecutor defines the interface accepted by the Exec command - provided for test stubbing
type RemoteExecutor interface {
Execute(
method string,
url *url.URL,
config *restclient.Config,
stdin io.Reader,
stdout, stderr io.Writer,
tty bool,
) error
}
// DefaultRemoteExecutor is the standard implementation of remote command execution
type DefaultRemoteExecutor struct{}
func (*DefaultRemoteExecutor) Execute(
method string,
url *url.URL,
config *restclient.Config,
stdin io.Reader,
stdout, stderr io.Writer,
tty bool,
) error {
exec, err := remotecommand.NewSPDYExecutor(config, method, url)
if err != nil {
return err
}
return exec.Stream(remotecommand.StreamOptions{
Stdin: stdin,
Stdout: stdout,
Stderr: stderr,
Tty: tty,
})
}
// AttachOptions declare the arguments accepted by the Attach command
type AttachOptions struct {
Namespace string
PodName string
ContainerName string
Command []string
Executor RemoteExecutor
Client *kubernetes.Clientset
Config *restclient.Config
}
// Run executes a validated remote execution against a pod.
func (p *AttachOptions) Run() error {
// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
pod, err := p.Client.CoreV1().Pods(p.Namespace).Get(context.TODO(), p.PodName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("couldn't get pod details: %w", err)
}
if pod.Status.Phase != api.PodRunning {
return fmt.Errorf(
"pod %q (on namespace %q) is not running and cannot execute commands; current phase is %q",
p.PodName, p.Namespace, pod.Status.Phase,
)
}
// Ending with a newline is important to actually run the script
stdin := strings.NewReader(strings.Join(p.Command, " ") + "\n")
req := p.Client.CoreV1().RESTClient().Post().
Resource("pods").
Name(pod.Name).
Namespace(pod.Namespace).
SubResource("attach").
VersionedParams(&api.PodAttachOptions{
Container: p.ContainerName,
Stdin: true,
Stdout: false,
Stderr: false,
TTY: false,
}, scheme.ParameterCodec)
return p.Executor.Execute(http.MethodPost, req.URL(), p.Config, stdin, nil, nil, false)
}
func (p *AttachOptions) ShouldRetry(times int, err error) bool {
return shouldRetryKubernetesError(times, err)
}
func shouldRetryKubernetesError(times int, err error) bool {
var statusError *kubeerrors.StatusError
if times < commandConnectFailureMaxTries &&
errors.As(err, &statusError) &&
statusError.ErrStatus.Code == http.StatusInternalServerError &&
statusError.ErrStatus.Message == errorDialingBackendEOFMessage {
return true
}
return false
}
// ExecOptions declare the arguments accepted by the Exec command
type ExecOptions struct {
Namespace string
PodName string
ContainerName string
Stdin bool
Command []string
In io.Reader
Out io.Writer
Err io.Writer
Executor RemoteExecutor
Client *kubernetes.Clientset
Config *restclient.Config
}
// Run executes a validated remote execution against a pod.
func (p *ExecOptions) Run() error {
// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
pod, err := p.Client.CoreV1().Pods(p.Namespace).Get(context.TODO(), p.PodName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("couldn't get pod details: %w", err)
}
if pod.Status.Phase != api.PodRunning {
return fmt.Errorf(
"pod %q (on namespace '%s') is not running and cannot execute commands; current phase is %q",
p.PodName, p.Namespace, pod.Status.Phase,
)
}
if p.ContainerName == "" {
logrus.Infof("defaulting container name to '%s'", pod.Spec.Containers[0].Name)
p.ContainerName = pod.Spec.Containers[0].Name
}
return p.executeRequest()
}
func (p *ExecOptions) executeRequest() error {
req := p.Client.CoreV1().RESTClient().Post().
Resource("pods").
Name(p.PodName).
Namespace(p.Namespace).
SubResource("exec").
Param("container", p.ContainerName)
var stdin io.Reader
if p.Stdin {
stdin = p.In
}
req.VersionedParams(&api.PodExecOptions{
Container: p.ContainerName,
Command: p.Command,
Stdin: stdin != nil,
Stdout: p.Out != nil,
Stderr: p.Err != nil,
}, scheme.ParameterCodec)
return p.Executor.Execute(http.MethodPost, req.URL(), p.Config, stdin, p.Out, p.Err, false)
}
func (p *ExecOptions) ShouldRetry(times int, err error) bool {
return shouldRetryKubernetesError(times, err)
}
func init() {
runtime.ErrorHandlers = append(runtime.ErrorHandlers, func(err error) {
logrus.WithError(err).Error("K8S stream error")
})
runtime.PanicHandlers = append(runtime.PanicHandlers, func(r interface{}) {
logrus.Errorf("K8S stream panic: %v", r)
})
}
package kubernetes
import (
"fmt"
"strings"
"unicode"
"github.com/hashicorp/go-version"
"k8s.io/client-go/kubernetes"
)
type featureChecker interface {
IsHostAliasSupported() (bool, error)
}
type kubeClientFeatureChecker struct {
kubeClient *kubernetes.Clientset
}
// https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/
var minimumHostAliasesVersionRequired, _ = version.NewVersion("1.7")
type badVersionError struct {
major string
minor string
inner error
}
func (s *badVersionError) Error() string {
return fmt.Sprintf("parsing Kubernetes version %s.%s - %s", s.major, s.minor, s.inner)
}
func (s *badVersionError) Is(err error) bool {
_, ok := err.(*badVersionError)
return ok
}
func (c *kubeClientFeatureChecker) IsHostAliasSupported() (bool, error) {
verInfo, err := c.kubeClient.ServerVersion()
if err != nil {
return false, err
}
major := cleanVersion(verInfo.Major)
minor := cleanVersion(verInfo.Minor)
ver, err := version.NewVersion(fmt.Sprintf("%s.%s", major, minor))
if err != nil {
// Use the original major and minor parts of the version so we can better see in the logs
// what came straight from kubernetes. The inner error from version.NewVersion will tell us
// what version we actually tried to parse
return false, &badVersionError{
major: verInfo.Major,
minor: verInfo.Minor,
inner: err,
}
}
supportsHostAliases := ver.GreaterThan(minimumHostAliasesVersionRequired) ||
ver.Equal(minimumHostAliasesVersionRequired)
return supportsHostAliases, nil
}
// Sometimes kubernetes returns a version which aren't valid semver versions
// or invalid enough that the version package can't parse them e.g. GCP returns 1.14+
func cleanVersion(version string) string {
// Try to find the index of the first symbol that isn't a digit
// use all the digits before that symbol as the version
nonDigitIndex := strings.IndexFunc(version, func(r rune) bool {
return !unicode.IsDigit(r)
})
if nonDigitIndex == -1 {
return version
}
return version[:nonDigitIndex]
}
package kubernetes
import (
"fmt"
api "k8s.io/api/core/v1"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/container/services"
"gitlab.com/gitlab-org/gitlab-runner/helpers/dns"
)
type invalidHostAliasDNSError struct {
service common.Image
inner error
}
func (e *invalidHostAliasDNSError) Error() string {
return fmt.Sprintf(
"provided host alias %s for service %s is invalid DNS. %s",
e.service.Alias,
e.service.Name,
e.inner,
)
}
func (e *invalidHostAliasDNSError) Is(err error) bool {
_, ok := err.(*invalidHostAliasDNSError)
return ok
}
func createHostAliases(services common.Services, hostAliases []api.HostAlias) ([]api.HostAlias, error) {
servicesHostAlias, err := createServicesHostAlias(services)
if err != nil {
return nil, err
}
// The order that we add host aliases matter here. The host file resolves
// host on a firs-come-first-served basis. We always want to have the
// service host aliases first so it resolves to that ip.
var allHostAliases []api.HostAlias
if servicesHostAlias != nil {
allHostAliases = append(allHostAliases, *servicesHostAlias)
}
allHostAliases = append(allHostAliases, hostAliases...)
return allHostAliases, nil
}
func createServicesHostAlias(srvs common.Services) (*api.HostAlias, error) {
var hostnames []string
for _, srv := range srvs {
// Services with ports are coming from .gitlab-webide.yml
// they are used for ports mapping and their aliases are in no way validated
// so we ignore them. Check out https://gitlab.com/gitlab-org/gitlab-runner/merge_requests/1170
// for details
if len(srv.Ports) > 0 {
continue
}
serviceMeta := services.SplitNameAndVersion(srv.Name)
for _, alias := range serviceMeta.Aliases {
// For backward compatibility reasons a non DNS1123 compliant alias might be generated,
// this will be removed in https://gitlab.com/gitlab-org/gitlab-runner/issues/6100
err := dns.ValidateDNS1123Subdomain(alias)
if err == nil {
hostnames = append(hostnames, alias)
}
}
if srv.Alias == "" {
continue
}
err := dns.ValidateDNS1123Subdomain(srv.Alias)
if err != nil {
return nil, &invalidHostAliasDNSError{service: srv, inner: err}
}
hostnames = append(hostnames, srv.Alias)
}
// no service hostnames to add to aliases
if len(hostnames) == 0 {
return nil, nil
}
return &api.HostAlias{IP: "127.0.0.1", Hostnames: hostnames}, nil
}
package pull
import "fmt"
// compile-time assertion to ensure ImagePullError always implements the
// error interface
var _ error = &ImagePullError{}
type ImagePullError struct {
Message string
Image string
}
func (e *ImagePullError) Error() string {
return fmt.Sprintf("pulling image %q: %s", e.Image, e.Message)
}
package pull
import (
"errors"
"fmt"
"sync"
api "k8s.io/api/core/v1"
)
// Manager defines the interface for a state machine which keeps track of the appropriate pull policy to use
// for each image definition
type Manager interface {
// GetPullPolicyFor returns the pull policy that should be used for the subsequent pull operation
// for the specified image
GetPullPolicyFor(image string) (api.PullPolicy, error)
// UpdatePolicyForImage updates the pull policy for the image designated in the specified error,
// and returns whether a new pull operation with a different pull policy can be attempted
UpdatePolicyForImage(attempt int, imagePullErr *ImagePullError) bool
}
type pullLogger interface {
Infoln(args ...interface{})
Warningln(args ...interface{})
}
type manager struct {
logger pullLogger
pullPolicies []api.PullPolicy
mu sync.RWMutex
failureMap map[string]int
}
func NewPullManager(pullPolicies []api.PullPolicy, logger pullLogger) Manager {
if len(pullPolicies) == 0 {
pullPolicies = []api.PullPolicy{""}
}
return &manager{
pullPolicies: pullPolicies,
failureMap: map[string]int{},
logger: logger,
}
}
func (m *manager) GetPullPolicyFor(image string) (api.PullPolicy, error) {
m.mu.RLock()
defer m.mu.RUnlock()
failureCount := m.failureMap[image]
if failureCount < len(m.pullPolicies) {
return m.pullPolicies[failureCount], nil
}
return "", errors.New("pull failed")
}
func (m *manager) UpdatePolicyForImage(attempt int, imagePullErr *ImagePullError) bool {
pullPolicy, _ := m.GetPullPolicyFor(imagePullErr.Image)
m.markPullFailureFor(imagePullErr.Image)
m.logger.Warningln(fmt.Sprintf(
"Failed to pull image with policy %q: %v",
pullPolicy,
imagePullErr.Message,
))
nextPullPolicy, errPull := m.GetPullPolicyFor(imagePullErr.Image)
if errPull == nil {
m.logger.Infoln(fmt.Sprintf(
"Attempt #%d: Trying %q pull policy for %q image",
attempt+1,
nextPullPolicy,
imagePullErr.Image,
))
return true
}
return false
}
// markPullFailureFor informs of a failure to pull the specified image
func (m *manager) markPullFailureFor(image string) {
m.mu.Lock()
defer m.mu.Unlock()
m.failureMap[image]++
}
package kubernetes
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"path"
"strings"
"sync"
"time"
"github.com/docker/cli/cli/config/types"
"github.com/jpillora/backoff"
"golang.org/x/net/context"
api "k8s.io/api/core/v1"
kubeerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/kubernetes"
_ "k8s.io/client-go/plugin/pkg/client/auth" // Register all available authentication methods
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/util/exec"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors"
"gitlab.com/gitlab-org/gitlab-runner/executors/kubernetes/internal/pull"
"gitlab.com/gitlab-org/gitlab-runner/helpers/container/helperimage"
"gitlab.com/gitlab-org/gitlab-runner/helpers/dns"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker/auth"
"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
"gitlab.com/gitlab-org/gitlab-runner/helpers/retry"
"gitlab.com/gitlab-org/gitlab-runner/session/proxy"
"gitlab.com/gitlab-org/gitlab-runner/shells"
)
const (
buildContainerName = "build"
helperContainerName = "helper"
detectShellScriptName = "detect_shell_script"
// The `.ps1` extension is added to the script name to fix a strange behavior
// where stage scripts wouldn't be executed otherwise
parsePwshScriptName = "parse_pwsh_script.ps1"
waitLogFileTimeout = time.Minute
outputLogFileNotExistsExitCode = 100
unknownLogProcessorExitCode = 1000
// nodeSelectorWindowsBuildLabel is the label used to reference a specific Windows Version.
// https://kubernetes.io/docs/reference/labels-annotations-taints/#nodekubernetesiowindows-build
nodeSelectorWindowsBuildLabel = "node.kubernetes.io/windows-build"
)
var (
executorOptions = executors.ExecutorOptions{
DefaultCustomBuildsDirEnabled: true,
DefaultBuildsDir: "/builds",
DefaultCacheDir: "/cache",
SharedBuildsDir: true,
Shell: common.ShellScriptInfo{
Shell: "bash",
Type: common.NormalShell,
RunnerCommand: "/usr/bin/gitlab-runner-helper",
},
ShowHostname: true,
}
errIncorrectShellType = fmt.Errorf("kubernetes executor incorrect shell type")
)
// GetDefaultCapDrop returns the default capabilities that should be dropped
// from a build container.
func GetDefaultCapDrop(os string) []string {
// windows does not support security context capabilities
if os == helperimage.OSTypeWindows {
return nil
}
return []string{
// Reasons for disabling NET_RAW by default were
// discussed in https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26833
"NET_RAW",
}
}
type commandTerminatedError struct {
exitCode int
}
func (c *commandTerminatedError) Error() string {
return fmt.Sprintf("command terminated with exit code %d", c.exitCode)
}
func (c *commandTerminatedError) Is(err error) bool {
_, ok := err.(*commandTerminatedError)
return ok
}
type podPhaseError struct {
name string
phase api.PodPhase
}
func (p *podPhaseError) Error() string {
return fmt.Sprintf("pod %q status is %q", p.name, p.phase)
}
type kubernetesOptions struct {
Image common.Image
Services common.Services
}
type executor struct {
executors.AbstractExecutor
kubeClient *kubernetes.Clientset
kubeConfig *restclient.Config
pod *api.Pod
configMap *api.ConfigMap
credentials *api.Secret
options *kubernetesOptions
services []api.Service
configurationOverwrites *overwrites
pullManager pull.Manager
helperImageInfo helperimage.Info
featureChecker featureChecker
newLogProcessor func() logProcessor
remoteProcessTerminated chan shells.TrapCommandExitStatus
// Flag if a repo mount and emptyDir volume are needed
requireDefaultBuildsDirVolume *bool
}
type serviceDeleteResponse struct {
serviceName string
err error
}
type serviceCreateResponse struct {
service *api.Service
err error
}
func (s *executor) Prepare(options common.ExecutorPrepareOptions) (err error) {
s.AbstractExecutor.PrepareConfiguration(options)
if err = s.prepareOverwrites(options.Build.GetAllVariables()); err != nil {
return fmt.Errorf("couldn't prepare overwrites: %w", err)
}
var pullPolicies []api.PullPolicy
if pullPolicies, err = s.Config.Kubernetes.GetPullPolicies(); err != nil {
return fmt.Errorf("couldn't get pull policy: %w", err)
}
s.pullManager = pull.NewPullManager(pullPolicies, &s.BuildLogger)
s.prepareOptions(options.Build)
if err = s.checkDefaults(); err != nil {
return fmt.Errorf("check defaults error: %w", err)
}
s.kubeConfig, err = getKubeClientConfig(s.Config.Kubernetes, s.configurationOverwrites)
if err != nil {
return fmt.Errorf("getting Kubernetes config: %w", err)
}
s.kubeClient, err = kubernetes.NewForConfig(s.kubeConfig)
if err != nil {
return fmt.Errorf("connecting to Kubernetes: %w", err)
}
s.helperImageInfo, err = s.prepareHelperImage()
if err != nil {
return fmt.Errorf("prepare helper image: %w", err)
}
// setup default executor options based on OS type
s.setupDefaultExecutorOptions(s.helperImageInfo.OSType)
s.featureChecker = &kubeClientFeatureChecker{kubeClient: s.kubeClient}
imageName := s.Build.GetAllVariables().ExpandValue(s.options.Image.Name)
s.Println("Using Kubernetes executor with image", imageName, "...")
if !s.Build.IsFeatureFlagOn(featureflags.UseLegacyKubernetesExecutionStrategy) {
s.Println("Using attach strategy to execute scripts...")
}
s.Debugln(fmt.Sprintf("Using helper image: %s:%s", s.helperImageInfo.Name, s.helperImageInfo.Tag))
err = s.AbstractExecutor.PrepareBuildAndShell()
if err != nil {
return fmt.Errorf("prepare build and shell: %w", err)
}
if s.BuildShell.PassFile {
return fmt.Errorf("kubernetes doesn't support shells that require script file")
}
return err
}
func (s *executor) setupDefaultExecutorOptions(os string) {
if os == helperimage.OSTypeWindows {
s.DefaultBuildsDir = `C:\builds`
s.DefaultCacheDir = `C:\cache`
s.ExecutorOptions.Shell.Shell = shells.SNPowershell
s.ExecutorOptions.Shell.RunnerCommand = "gitlab-runner-helper"
}
}
func (s *executor) prepareHelperImage() (helperimage.Info, error) {
config := helperimage.Config{
OSType: helperimage.OSTypeLinux,
Architecture: "amd64",
GitLabRegistry: s.Build.IsFeatureFlagOn(featureflags.GitLabRegistryHelperImage),
Shell: s.Config.Shell,
Flavor: s.Config.Kubernetes.HelperImageFlavor,
}
// use node selector labels to better select the correct image
if s.Config.Kubernetes.NodeSelector != nil {
for label, option := range map[string]*string{
api.LabelArchStable: &config.Architecture,
api.LabelOSStable: &config.OSType,
nodeSelectorWindowsBuildLabel: &config.OperatingSystem,
} {
value := s.Config.Kubernetes.NodeSelector[label]
if value != "" {
*option = value
}
}
}
return helperimage.Get(common.REVISION, config)
}
func (s *executor) Run(cmd common.ExecutorCommand) error {
for attempt := 1; ; attempt++ {
var err error
if s.Build.IsFeatureFlagOn(featureflags.UseLegacyKubernetesExecutionStrategy) {
s.Debugln("Starting Kubernetes command...")
err = s.runWithExecLegacy(cmd)
} else {
s.Debugln("Starting Kubernetes command with attach...")
err = s.runWithAttach(cmd)
}
var imagePullErr *pull.ImagePullError
if errors.As(err, &imagePullErr) {
if s.pullManager.UpdatePolicyForImage(attempt, imagePullErr) {
s.cleanupResources()
s.pod = nil
continue
}
}
return err
}
}
func (s *executor) runWithExecLegacy(cmd common.ExecutorCommand) error {
if s.pod == nil {
err := s.setupCredentials()
if err != nil {
return err
}
err = s.setupBuildPod(nil)
if err != nil {
return err
}
}
containerName := buildContainerName
containerCommand := s.BuildShell.DockerCommand
if cmd.Predefined {
containerName = helperContainerName
containerCommand = s.helperImageInfo.Cmd
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
s.Debugln(fmt.Sprintf(
"Starting in container %q the command %q with script: %s",
containerName,
containerCommand,
cmd.Script,
))
select {
case err := <-s.runInContainerWithExecLegacy(ctx, containerName, containerCommand, cmd.Script):
s.Debugln(fmt.Sprintf("Container %q exited with error: %v", containerName, err))
var exitError exec.CodeExitError
if err != nil && errors.As(err, &exitError) {
return &common.BuildError{Inner: err, ExitCode: exitError.ExitStatus()}
}
return err
case <-cmd.Context.Done():
return fmt.Errorf("build aborted")
}
}
func (s *executor) runWithAttach(cmd common.ExecutorCommand) error {
err := s.ensurePodsConfigured(cmd.Context)
if err != nil {
return err
}
ctx, cancel := context.WithCancel(cmd.Context)
defer cancel()
containerName, containerCommand := s.getContainerInfo(cmd)
s.Debugln(fmt.Sprintf(
"Starting in container %q the command %q with script: %s",
containerName,
containerCommand,
cmd.Script,
))
podStatusCh := s.watchPodStatus(ctx)
select {
case err := <-s.runInContainer(containerName, containerCommand):
s.Debugln(fmt.Sprintf("Container %q exited with error: %v", containerName, err))
var terminatedError *commandTerminatedError
if err != nil && errors.As(err, &terminatedError) {
return &common.BuildError{Inner: err, ExitCode: terminatedError.exitCode}
}
return err
case err := <-podStatusCh:
if IsKubernetesPodNotFoundError(err) {
return err
}
return &common.BuildError{Inner: err}
case <-ctx.Done():
return fmt.Errorf("build aborted")
}
}
func (s *executor) ensurePodsConfigured(ctx context.Context) error {
if s.pod != nil {
return nil
}
err := s.setupCredentials()
if err != nil {
return fmt.Errorf("setting up credentials: %w", err)
}
err = s.setupScriptsConfigMap()
if err != nil {
return fmt.Errorf("setting up scripts configMap: %w", err)
}
permissionsInitContainer, err := s.buildLogPermissionsInitContainer()
if err != nil {
return fmt.Errorf("building log permissions init container: %w", err)
}
err = s.setupBuildPod([]api.Container{permissionsInitContainer})
if err != nil {
return fmt.Errorf("setting up build pod: %w", err)
}
status, err := waitForPodRunning(ctx, s.kubeClient, s.pod, s.Trace, s.Config.Kubernetes)
if err != nil {
return fmt.Errorf("waiting for pod running: %w", err)
}
if status != api.PodRunning {
return fmt.Errorf("pod failed to enter running state: %s", status)
}
go s.processLogs(ctx)
return nil
}
func (s *executor) getContainerInfo(cmd common.ExecutorCommand) (string, []string) {
var containerCommand []string
containerName := buildContainerName
switch s.Shell().Shell {
case shells.SNPwsh:
// Translates to roughly "/path/to/parse_pwsh_script.ps1 /path/to/stage_script /path/to/logFile"
containerCommand = []string{
s.scriptPath(parsePwshScriptName),
s.scriptPath(cmd.Stage),
s.logFile(),
s.buildRedirectionCmd(),
}
if cmd.Predefined {
containerName = helperContainerName
containerCommand = []string{fmt.Sprintf("Get-Content -Path %s | ", s.scriptPath(cmd.Stage))}
containerCommand = append(containerCommand, s.helperImageInfo.Cmd...)
containerCommand = append(containerCommand, s.buildRedirectionCmd())
}
default:
// Translates to roughly "sh /detect/shell/path.sh /stage/script/path.sh"
// which when the detect shell exits becomes something like "bash /stage/script/path.sh".
// This works unlike "gitlab-runner-build" since the detect shell passes arguments with "$@"
containerCommand = []string{
"sh",
s.scriptPath(detectShellScriptName),
s.scriptPath(cmd.Stage),
s.buildRedirectionCmd(),
}
if cmd.Predefined {
containerName = helperContainerName
// We use redirection here since the "gitlab-runner-build" helper doesn't pass input args
// to the shell it executes, so we technically pass the script to the stdin of the underlying shell
// translates roughly to "gitlab-runner-build <<< /stage/script/path.sh"
containerCommand = append(
s.helperImageInfo.Cmd,
"<<<",
s.scriptPath(cmd.Stage),
s.buildRedirectionCmd(),
)
}
}
return containerName, containerCommand
}
func (s *executor) buildLogPermissionsInitContainer() (api.Container, error) {
// We need to create the log file in which all scripts will append their output.
// The log file is created with the current user. There are 3 different scenarios for the user:
// 1. The user in all images and containers is root, in that case the chmod is redundant since they
// will all have permissions to the file.
// 2. The user of the helper image is root, however the build image's user is not root.
// In that case we need to allow the build user to write to the log file from inside the
// build container. That's where the chmod comes into play.
// 3. No user is root but all containers have the same user ID. In that case create the file.
// It will have the same user and group owner across all containers. This is the case for Kubernetes
// where the PodSecurityContext is set manually or for Openshift where each pod has a different user ID.
// *4. We don't allow setting different user IDs across containers, if that ever becomes the case
// we might need to try and chown the log file for the group only.
logFile := s.logFile()
chmod := fmt.Sprintf("touch %s && (chmod 777 %s || exit 0)", logFile, logFile)
pullPolicy, err := s.pullManager.GetPullPolicyFor(s.getHelperImage())
if err != nil {
return api.Container{}, fmt.Errorf("getting pull policy for log permissions init container: %w", err)
}
return api.Container{
Name: "init-logs",
Image: s.getHelperImage(),
Command: []string{"sh", "-c", chmod},
VolumeMounts: s.getVolumeMounts(),
ImagePullPolicy: pullPolicy,
}, nil
}
func (s *executor) buildRedirectionCmd() string {
return fmt.Sprintf("2>&1 | tee -a %s", s.logFile())
}
func (s *executor) processLogs(ctx context.Context) {
processor := s.newLogProcessor()
logsCh, errCh := processor.Process(ctx)
for {
select {
case line, ok := <-logsCh:
if !ok {
return
}
var status shells.TrapCommandExitStatus
if status.TryUnmarshal(line) {
s.remoteProcessTerminated <- status
continue
}
_, err := s.Trace.Write(append([]byte(line), '\n'))
if err != nil {
s.Warningln(fmt.Sprintf("Error writing log line to trace: %v", err))
}
case err, ok := <-errCh:
if !ok {
continue
}
exitCode := getExitCode(err)
s.Warningln(fmt.Sprintf("%v", err))
// Script can be kept to nil as not being used after the exitStatus is received L1223
s.remoteProcessTerminated <- shells.TrapCommandExitStatus{CommandExitCode: &exitCode}
}
}
}
// getExitCode tries to extract the exit code from an inner exec.CodeExitError
// This error may be returned by the underlying kubernetes connection stream
// however it's not guaranteed to be.
// getExitCode would return unknownLogProcessorExitCode if err isn't of type exec.CodeExitError
// or if it's nil
func getExitCode(err error) int {
var exitErr exec.CodeExitError
if errors.As(err, &exitErr) {
return exitErr.Code
}
return unknownLogProcessorExitCode
}
func (s *executor) setupScriptsConfigMap() error {
s.Debugln("Setting up scripts config map")
// After issue https://gitlab.com/gitlab-org/gitlab-runner/issues/10342 is resolved and
// the legacy execution mode is removed we can remove the manual construction of trapShell and just use "bash+trap"
// in the exec options
shell, err := s.retrieveShell()
if err != nil {
return err
}
scripts, err := s.generateScripts(shell)
if err != nil {
return err
}
configMap := &api.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("%s-scripts", s.Build.ProjectUniqueName()),
Namespace: s.configurationOverwrites.namespace,
},
Data: scripts,
}
// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
s.configMap, err = s.kubeClient.
CoreV1().
ConfigMaps(s.configurationOverwrites.namespace).
Create(context.TODO(), configMap, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("generating scripts config map: %w", err)
}
return nil
}
func (s *executor) retrieveShell() (common.Shell, error) {
bashShell, ok := common.GetShell(s.Shell().Shell).(*shells.BashShell)
if ok {
return &shells.BashTrapShell{BashShell: bashShell, LogFile: s.logFile()}, nil
}
pwshShell, ok := common.GetShell(s.Shell().Shell).(*shells.PowerShell)
if ok {
return &shells.PwshTrapShell{PowerShell: pwshShell, LogFile: s.logFile()}, nil
}
return nil, errIncorrectShellType
}
func (s *executor) generateScripts(shell common.Shell) (map[string]string, error) {
scripts := map[string]string{}
switch s.Shell().Shell {
case shells.SNPwsh:
scripts[parsePwshScriptName] = shells.PwshValidationScript
default:
scripts[detectShellScriptName] = shells.BashDetectShellScript
}
for _, stage := range s.Build.BuildStages() {
script, err := shell.GenerateScript(stage, *s.Shell())
if errors.Is(err, common.ErrSkipBuildStage) {
continue
} else if err != nil {
return nil, fmt.Errorf("generating trap shell script: %w", err)
}
scripts[string(stage)] = script
}
return scripts, nil
}
func (s *executor) Finish(err error) {
if IsKubernetesPodNotFoundError(err) {
// Avoid an additional error message when trying to
// cleanup a pod that we know no longer exists
s.pod = nil
}
s.AbstractExecutor.Finish(err)
}
func (s *executor) Cleanup() {
s.cleanupResources()
closeKubeClient(s.kubeClient)
s.AbstractExecutor.Cleanup()
}
func (s *executor) cleanupServices() {
ch := make(chan serviceDeleteResponse)
var wg sync.WaitGroup
wg.Add(len(s.services))
for _, service := range s.services {
go s.deleteKubernetesService(service.ObjectMeta.Name, ch, &wg)
}
go func() {
wg.Wait()
close(ch)
}()
for res := range ch {
if res.err != nil {
s.Errorln(fmt.Sprintf("Error cleaning up the pod service %q: %v", res.serviceName, res.err))
}
}
}
func (s *executor) deleteKubernetesService(serviceName string, ch chan<- serviceDeleteResponse, wg *sync.WaitGroup) {
defer wg.Done()
// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
err := s.kubeClient.CoreV1().
Services(s.configurationOverwrites.namespace).
Delete(context.TODO(), serviceName, metav1.DeleteOptions{})
ch <- serviceDeleteResponse{serviceName: serviceName, err: err}
}
func (s *executor) cleanupResources() {
if s.pod != nil {
// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
err := s.kubeClient.
CoreV1().
Pods(s.pod.Namespace).
Delete(context.TODO(), s.pod.Name, metav1.DeleteOptions{})
if err != nil {
s.Errorln(fmt.Sprintf("Error cleaning up pod: %s", err.Error()))
}
}
if s.credentials != nil {
// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
err := s.kubeClient.CoreV1().
Secrets(s.configurationOverwrites.namespace).
Delete(context.TODO(), s.credentials.Name, metav1.DeleteOptions{})
if err != nil {
s.Errorln(fmt.Sprintf("Error cleaning up secrets: %s", err.Error()))
}
}
if s.configMap != nil {
// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
err := s.kubeClient.CoreV1().
ConfigMaps(s.configurationOverwrites.namespace).
Delete(context.TODO(), s.configMap.Name, metav1.DeleteOptions{})
if err != nil {
s.Errorln(fmt.Sprintf("Error cleaning up configmap: %s", err.Error()))
}
}
s.cleanupServices()
}
//nolint:funlen
func (s *executor) buildContainer(
name, image string,
imageDefinition common.Image,
requests, limits api.ResourceList,
containerCommand ...string,
) (api.Container, error) {
// check if the image/service is allowed
internalImages := []string{
s.Config.Kubernetes.Image,
s.helperImageInfo.Name,
}
var (
optionName string
allowedImages []string
)
if strings.HasPrefix(name, "svc-") {
optionName = "services"
allowedImages = s.Config.Kubernetes.AllowedServices
} else if name == buildContainerName {
optionName = "images"
allowedImages = s.Config.Kubernetes.AllowedImages
}
verifyAllowedImageOptions := common.VerifyAllowedImageOptions{
Image: image,
OptionName: optionName,
AllowedImages: allowedImages,
InternalImages: internalImages,
}
err := common.VerifyAllowedImage(verifyAllowedImageOptions, s.BuildLogger)
if err != nil {
return api.Container{}, err
}
containerPorts := make([]api.ContainerPort, len(imageDefinition.Ports))
proxyPorts := make([]proxy.Port, len(imageDefinition.Ports))
for i, port := range imageDefinition.Ports {
proxyPorts[i] = proxy.Port{Name: port.Name, Number: port.Number, Protocol: port.Protocol}
containerPorts[i] = api.ContainerPort{ContainerPort: int32(port.Number)}
}
if len(proxyPorts) > 0 {
serviceName := imageDefinition.Alias
if serviceName == "" {
serviceName = name
if name != buildContainerName {
serviceName = fmt.Sprintf("proxy-%s", name)
}
}
s.ProxyPool[serviceName] = s.newProxy(serviceName, proxyPorts)
}
pullPolicy, err := s.pullManager.GetPullPolicyFor(image)
if err != nil {
return api.Container{}, err
}
command, args := s.getCommandAndArgs(imageDefinition, containerCommand...)
container := api.Container{
Name: name,
Image: image,
ImagePullPolicy: pullPolicy,
Command: command,
Args: args,
Env: buildVariables(s.Build.GetAllVariables().PublicOrInternal()),
Resources: api.ResourceRequirements{
Limits: limits,
Requests: requests,
},
Ports: containerPorts,
VolumeMounts: s.getVolumeMounts(),
SecurityContext: &api.SecurityContext{
Privileged: s.Config.Kubernetes.Privileged,
AllowPrivilegeEscalation: s.Config.Kubernetes.AllowPrivilegeEscalation,
Capabilities: getCapabilities(
GetDefaultCapDrop(s.helperImageInfo.OSType),
s.Config.Kubernetes.CapAdd,
s.Config.Kubernetes.CapDrop,
),
},
Stdin: true,
}
return container, nil
}
func (s *executor) getCommandAndArgs(imageDefinition common.Image, command ...string) ([]string, []string) {
if len(command) == 0 && len(imageDefinition.Entrypoint) > 0 {
command = imageDefinition.Entrypoint
}
var args []string
if len(imageDefinition.Command) > 0 {
args = imageDefinition.Command
}
return command, args
}
func (s *executor) logFile() string {
return path.Join(s.logsDir(), "output.log")
}
func (s *executor) logsDir() string {
return fmt.Sprintf("/logs-%d-%d", s.Build.JobInfo.ProjectID, s.Build.JobResponse.ID)
}
func (s *executor) scriptsDir() string {
return fmt.Sprintf("/scripts-%d-%d", s.Build.JobInfo.ProjectID, s.Build.JobResponse.ID)
}
func (s *executor) scriptPath(stage common.BuildStage) string {
return path.Join(s.scriptsDir(), string(stage))
}
func (s *executor) getVolumeMounts() []api.VolumeMount {
var mounts []api.VolumeMount
// The configMap is nil when using legacy execution
if s.configMap != nil {
// These volume mounts **MUST NOT** be mounted inside another volume mount.
// E.g. mounting them inside the "repo" volume mount will cause the whole volume
// to be owned by root instead of the current user of the image. Something similar
// is explained here https://github.com/kubernetes/kubernetes/issues/2630#issuecomment-64679120
// where the first container determines the ownership of a volume. However, it seems like
// when mounting a volume inside another volume the first container or the first point of contact
// becomes root, regardless of SecurityContext or Image settings changing the user ID of the container.
// This causes builds to stop working in environments such as OpenShift where there's no root access
// resulting in an inability to modify anything inside the parent volume.
mounts = append(
mounts,
api.VolumeMount{
Name: "scripts",
MountPath: s.scriptsDir(),
},
api.VolumeMount{
Name: "logs",
MountPath: s.logsDir(),
})
}
mounts = append(mounts, s.getVolumeMountsForConfig()...)
if s.isDefaultBuildsDirVolumeRequired() {
mounts = append(mounts, api.VolumeMount{
Name: "repo",
MountPath: s.Build.RootDir,
})
}
return mounts
}
func (s *executor) getVolumeMountsForConfig() []api.VolumeMount {
var mounts []api.VolumeMount
for _, mount := range s.Config.Kubernetes.Volumes.HostPaths {
mounts = append(mounts, api.VolumeMount{
Name: mount.Name,
MountPath: mount.MountPath,
SubPath: mount.SubPath,
ReadOnly: mount.ReadOnly,
})
}
for _, mount := range s.Config.Kubernetes.Volumes.Secrets {
mounts = append(mounts, api.VolumeMount{
Name: mount.Name,
MountPath: mount.MountPath,
SubPath: mount.SubPath,
ReadOnly: mount.ReadOnly,
})
}
for _, mount := range s.Config.Kubernetes.Volumes.PVCs {
mounts = append(mounts, api.VolumeMount{
Name: mount.Name,
MountPath: mount.MountPath,
SubPath: mount.SubPath,
ReadOnly: mount.ReadOnly,
})
}
for _, mount := range s.Config.Kubernetes.Volumes.ConfigMaps {
mounts = append(mounts, api.VolumeMount{
Name: mount.Name,
MountPath: mount.MountPath,
SubPath: mount.SubPath,
ReadOnly: mount.ReadOnly,
})
}
for _, mount := range s.Config.Kubernetes.Volumes.EmptyDirs {
mounts = append(mounts, api.VolumeMount{
Name: mount.Name,
MountPath: mount.MountPath,
SubPath: mount.SubPath,
})
}
for _, mount := range s.Config.Kubernetes.Volumes.CSIs {
mounts = append(mounts, api.VolumeMount{
Name: mount.Name,
MountPath: mount.MountPath,
SubPath: mount.SubPath,
ReadOnly: mount.ReadOnly,
})
}
return mounts
}
func (s *executor) getVolumes() []api.Volume {
volumes := s.getVolumesForConfig()
if s.isDefaultBuildsDirVolumeRequired() {
volumes = append(volumes, api.Volume{
Name: "repo",
VolumeSource: api.VolumeSource{
EmptyDir: &api.EmptyDirVolumeSource{},
},
})
}
// The configMap is nil when using legacy execution
if s.configMap == nil {
return volumes
}
mode := int32(0777)
optional := false
volumes = append(
volumes,
api.Volume{
Name: "scripts",
VolumeSource: api.VolumeSource{
ConfigMap: &api.ConfigMapVolumeSource{
LocalObjectReference: api.LocalObjectReference{
Name: s.configMap.Name,
},
DefaultMode: &mode,
Optional: &optional,
},
},
},
api.Volume{
Name: "logs",
VolumeSource: api.VolumeSource{
EmptyDir: &api.EmptyDirVolumeSource{},
},
})
return volumes
}
func (s *executor) getVolumesForConfig() []api.Volume {
var volumes []api.Volume
volumes = append(volumes, s.getVolumesForHostPaths()...)
volumes = append(volumes, s.getVolumesForSecrets()...)
volumes = append(volumes, s.getVolumesForPVCs()...)
volumes = append(volumes, s.getVolumesForConfigMaps()...)
volumes = append(volumes, s.getVolumesForEmptyDirs()...)
volumes = append(volumes, s.getVolumesForCSIs()...)
return volumes
}
func (s *executor) getVolumesForHostPaths() []api.Volume {
var volumes []api.Volume
for _, volume := range s.Config.Kubernetes.Volumes.HostPaths {
path := volume.HostPath
// Make backward compatible with syntax introduced in version 9.3.0
if path == "" {
path = volume.MountPath
}
volumes = append(volumes, api.Volume{
Name: volume.Name,
VolumeSource: api.VolumeSource{
HostPath: &api.HostPathVolumeSource{
Path: path,
},
},
})
}
return volumes
}
func (s *executor) getVolumesForSecrets() []api.Volume {
var volumes []api.Volume
for _, volume := range s.Config.Kubernetes.Volumes.Secrets {
var items []api.KeyToPath
for key, path := range volume.Items {
items = append(items, api.KeyToPath{Key: key, Path: path})
}
volumes = append(volumes, api.Volume{
Name: volume.Name,
VolumeSource: api.VolumeSource{
Secret: &api.SecretVolumeSource{
SecretName: volume.Name,
Items: items,
},
},
})
}
return volumes
}
func (s *executor) getVolumesForPVCs() []api.Volume {
var volumes []api.Volume
for _, volume := range s.Config.Kubernetes.Volumes.PVCs {
volumes = append(volumes, api.Volume{
Name: volume.Name,
VolumeSource: api.VolumeSource{
PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{
ClaimName: volume.Name,
ReadOnly: volume.ReadOnly,
},
},
})
}
return volumes
}
func (s *executor) getVolumesForConfigMaps() []api.Volume {
var volumes []api.Volume
for _, volume := range s.Config.Kubernetes.Volumes.ConfigMaps {
var items []api.KeyToPath
for key, path := range volume.Items {
items = append(items, api.KeyToPath{Key: key, Path: path})
}
volumes = append(volumes, api.Volume{
Name: volume.Name,
VolumeSource: api.VolumeSource{
ConfigMap: &api.ConfigMapVolumeSource{
LocalObjectReference: api.LocalObjectReference{
Name: volume.Name,
},
Items: items,
},
},
})
}
return volumes
}
func (s *executor) getVolumesForEmptyDirs() []api.Volume {
var volumes []api.Volume
for _, volume := range s.Config.Kubernetes.Volumes.EmptyDirs {
volumes = append(volumes, api.Volume{
Name: volume.Name,
VolumeSource: api.VolumeSource{
EmptyDir: &api.EmptyDirVolumeSource{
Medium: api.StorageMedium(volume.Medium),
},
},
})
}
return volumes
}
func (s *executor) getVolumesForCSIs() []api.Volume {
var volumes []api.Volume
for _, volume := range s.Config.Kubernetes.Volumes.CSIs {
volumes = append(volumes, api.Volume{
Name: volume.Name,
VolumeSource: api.VolumeSource{
CSI: &api.CSIVolumeSource{
Driver: volume.Driver,
FSType: &volume.FSType,
ReadOnly: &volume.ReadOnly,
VolumeAttributes: volume.VolumeAttributes,
},
},
})
}
return volumes
}
func (s *executor) isDefaultBuildsDirVolumeRequired() bool {
if s.requireDefaultBuildsDirVolume != nil {
return *s.requireDefaultBuildsDirVolume
}
var required = true
for _, mount := range s.getVolumeMountsForConfig() {
if mount.MountPath == s.Build.RootDir {
required = false
break
}
}
s.requireDefaultBuildsDirVolume = &required
return required
}
func (s *executor) setupCredentials() error {
s.Debugln("Setting up secrets")
authConfigs, err := auth.ResolveConfigs(s.Build.GetDockerAuthConfig(), s.Shell().User, s.Build.Credentials)
if err != nil {
return err
}
if len(authConfigs) == 0 {
return nil
}
dockerCfgs := make(map[string]types.AuthConfig)
for registry, registryInfo := range authConfigs {
dockerCfgs[registry] = registryInfo.AuthConfig
}
dockerCfgContent, err := json.Marshal(dockerCfgs)
if err != nil {
return err
}
secret := api.Secret{}
secret.GenerateName = s.Build.ProjectUniqueName()
secret.Namespace = s.configurationOverwrites.namespace
secret.Type = api.SecretTypeDockercfg
secret.Data = map[string][]byte{}
secret.Data[api.DockerConfigKey] = dockerCfgContent
// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
creds, err := s.kubeClient.
CoreV1().
Secrets(s.configurationOverwrites.namespace).
Create(context.TODO(), &secret, metav1.CreateOptions{})
if err != nil {
return err
}
s.credentials = creds
return nil
}
func (s *executor) getHostAliases() ([]api.HostAlias, error) {
supportsHostAliases, err := s.featureChecker.IsHostAliasSupported()
switch {
case errors.Is(err, &badVersionError{}):
s.Warningln("Checking for host alias support. Host aliases will be disabled.", err)
return nil, nil
case err != nil:
return nil, err
case !supportsHostAliases:
return nil, nil
}
return createHostAliases(s.options.Services, s.Config.Kubernetes.GetHostAliases())
}
//nolint:funlen
func (s *executor) setupBuildPod(initContainers []api.Container) error {
s.Debugln("Setting up build pod")
podServices := make([]api.Container, len(s.options.Services))
for i, service := range s.options.Services {
resolvedImage := s.Build.GetAllVariables().ExpandValue(service.Name)
var err error
podServices[i], err = s.buildContainer(
fmt.Sprintf("svc-%d", i),
resolvedImage,
service,
s.configurationOverwrites.serviceRequests,
s.configurationOverwrites.serviceLimits,
)
if err != nil {
return err
}
}
// We set a default label to the pod. This label will be used later
// by the services, to link each service to the pod
labels := map[string]string{"pod": s.Build.ProjectUniqueName()}
for k, v := range s.Build.Runner.Kubernetes.PodLabels {
labels[k] = s.Build.Variables.ExpandValue(v)
}
annotations := make(map[string]string)
for key, val := range s.configurationOverwrites.podAnnotations {
annotations[key] = s.Build.Variables.ExpandValue(val)
}
var imagePullSecrets []api.LocalObjectReference
for _, imagePullSecret := range s.Config.Kubernetes.ImagePullSecrets {
imagePullSecrets = append(imagePullSecrets, api.LocalObjectReference{Name: imagePullSecret})
}
if s.credentials != nil {
imagePullSecrets = append(imagePullSecrets, api.LocalObjectReference{Name: s.credentials.Name})
}
hostAliases, err := s.getHostAliases()
if err != nil {
return err
}
podConfig, err :=
s.preparePodConfig(labels, annotations, podServices, imagePullSecrets, hostAliases, initContainers)
if err != nil {
return err
}
s.Debugln("Creating build pod")
// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
pod, err := s.kubeClient.
CoreV1().
Pods(s.configurationOverwrites.namespace).
Create(context.TODO(), &podConfig, metav1.CreateOptions{})
if err != nil {
return err
}
s.pod = pod
s.services, err = s.makePodProxyServices()
if err != nil {
return err
}
return nil
}
//nolint:funlen
func (s *executor) preparePodConfig(
labels, annotations map[string]string,
services []api.Container,
imagePullSecrets []api.LocalObjectReference,
hostAliases []api.HostAlias,
initContainers []api.Container,
) (api.Pod, error) {
buildImage := s.Build.GetAllVariables().ExpandValue(s.options.Image.Name)
buildContainer, err := s.buildContainer(
buildContainerName,
buildImage,
s.options.Image,
s.configurationOverwrites.buildRequests,
s.configurationOverwrites.buildLimits,
s.BuildShell.DockerCommand...,
)
if err != nil {
return api.Pod{}, fmt.Errorf("building build container: %w", err)
}
helperContainer, err := s.buildContainer(
helperContainerName,
s.getHelperImage(),
common.Image{},
s.configurationOverwrites.helperRequests,
s.configurationOverwrites.helperLimits,
s.BuildShell.DockerCommand...,
)
if err != nil {
return api.Pod{}, fmt.Errorf("building helper container: %w", err)
}
pod := api.Pod{
ObjectMeta: metav1.ObjectMeta{
GenerateName: s.Build.ProjectUniqueName(),
Namespace: s.configurationOverwrites.namespace,
Labels: labels,
Annotations: annotations,
},
Spec: api.PodSpec{
Volumes: s.getVolumes(),
ServiceAccountName: s.configurationOverwrites.serviceAccount,
RestartPolicy: api.RestartPolicyNever,
NodeSelector: s.Config.Kubernetes.NodeSelector,
Tolerations: s.Config.Kubernetes.GetNodeTolerations(),
InitContainers: initContainers,
Containers: append([]api.Container{
buildContainer,
helperContainer,
}, services...),
TerminationGracePeriodSeconds: &s.Config.Kubernetes.TerminationGracePeriodSeconds,
ImagePullSecrets: imagePullSecrets,
SecurityContext: s.Config.Kubernetes.GetPodSecurityContext(),
HostAliases: hostAliases,
Affinity: s.Config.Kubernetes.GetAffinity(),
DNSPolicy: s.getDNSPolicy(),
DNSConfig: s.Config.Kubernetes.GetDNSConfig(),
},
}
return pod, nil
}
func (s *executor) getDNSPolicy() api.DNSPolicy {
dnsPolicy, err := s.Config.Kubernetes.DNSPolicy.Get()
if err != nil {
s.Warningln(fmt.Sprintf("falling back to cluster's default policy: %v", err))
}
return dnsPolicy
}
func (s *executor) getHelperImage() string {
if len(s.Config.Kubernetes.HelperImage) > 0 {
return common.AppVersion.Variables().ExpandValue(s.Config.Kubernetes.HelperImage)
}
if !s.Build.IsFeatureFlagOn(featureflags.GitLabRegistryHelperImage) {
s.Warningln(helperimage.DockerHubWarningMessage)
}
return s.helperImageInfo.String()
}
func (s *executor) makePodProxyServices() ([]api.Service, error) {
s.Debugln("Creating pod proxy services")
ch := make(chan serviceCreateResponse)
var wg sync.WaitGroup
wg.Add(len(s.ProxyPool))
for serviceName, serviceProxy := range s.ProxyPool {
serviceName = dns.MakeRFC1123Compatible(serviceName)
servicePorts := make([]api.ServicePort, len(serviceProxy.Settings.Ports))
for i, port := range serviceProxy.Settings.Ports {
// When there is more than one port Kubernetes requires a port name
portName := fmt.Sprintf("%s-%d", serviceName, port.Number)
servicePorts[i] = api.ServicePort{
Port: int32(port.Number),
TargetPort: intstr.FromInt(port.Number),
Name: portName,
}
}
serviceConfig := s.prepareServiceConfig(serviceName, servicePorts)
go s.createKubernetesService(&serviceConfig, serviceProxy.Settings, ch, &wg)
}
go func() {
wg.Wait()
close(ch)
}()
var proxyServices []api.Service
for res := range ch {
if res.err != nil {
err := fmt.Errorf("error creating the proxy service %q: %w", res.service.Name, res.err)
s.Errorln(err)
return []api.Service{}, err
}
proxyServices = append(proxyServices, *res.service)
}
return proxyServices, nil
}
func (s *executor) prepareServiceConfig(name string, ports []api.ServicePort) api.Service {
return api.Service{
ObjectMeta: metav1.ObjectMeta{
GenerateName: name,
Namespace: s.configurationOverwrites.namespace,
},
Spec: api.ServiceSpec{
Ports: ports,
Selector: map[string]string{"pod": s.Build.ProjectUniqueName()},
Type: api.ServiceTypeClusterIP,
},
}
}
func (s *executor) createKubernetesService(
service *api.Service,
proxySettings *proxy.Settings,
ch chan<- serviceCreateResponse,
wg *sync.WaitGroup,
) {
defer wg.Done()
// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
service, err := s.kubeClient.
CoreV1().
Services(s.pod.Namespace).
Create(context.TODO(), service, metav1.CreateOptions{})
if err == nil {
// Updating the internal service name reference and activating the proxy
proxySettings.ServiceName = service.Name
}
ch <- serviceCreateResponse{service: service, err: err}
}
func (s *executor) watchPodStatus(ctx context.Context) <-chan error {
// Buffer of 1 in case the context is cancelled while the timer tick case is being executed
// and the consumer is no longer reading from the channel while we try to write to it
ch := make(chan error, 1)
go func() {
defer close(ch)
t := time.NewTicker(time.Duration(s.Config.Kubernetes.GetPollInterval()) * time.Second)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
err := s.checkPodStatus()
if err != nil {
ch <- err
return
}
}
}
}()
return ch
}
func (s *executor) checkPodStatus() error {
// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
pod, err := s.kubeClient.
CoreV1().
Pods(s.pod.Namespace).
Get(context.TODO(), s.pod.Name, metav1.GetOptions{})
if IsKubernetesPodNotFoundError(err) {
return err
}
if err != nil {
// General request failure
s.Warningln("Getting job pod status", err)
return nil
}
if pod.Status.Phase != api.PodRunning {
return &podPhaseError{
name: s.pod.Name,
phase: pod.Status.Phase,
}
}
return nil
}
func (s *executor) runInContainer(name string, command []string) <-chan error {
errCh := make(chan error, 1)
go func() {
defer close(errCh)
attach := AttachOptions{
PodName: s.pod.Name,
Namespace: s.pod.Namespace,
ContainerName: name,
Command: command,
Config: s.kubeConfig,
Client: s.kubeClient,
Executor: &DefaultRemoteExecutor{},
}
retryable := retry.New(retry.WithBuildLog(&attach, &s.BuildLogger))
err := retryable.Run()
if err != nil {
errCh <- err
}
exitStatus := <-s.remoteProcessTerminated
if *exitStatus.CommandExitCode == 0 {
errCh <- nil
return
}
errCh <- &commandTerminatedError{exitCode: *exitStatus.CommandExitCode}
}()
return errCh
}
func (s *executor) runInContainerWithExecLegacy(
ctx context.Context,
name string,
command []string,
script string,
) <-chan error {
errCh := make(chan error, 1)
go func() {
defer close(errCh)
status, err := waitForPodRunning(ctx, s.kubeClient, s.pod, s.Trace, s.Config.Kubernetes)
if err != nil {
errCh <- err
return
}
if status != api.PodRunning {
errCh <- fmt.Errorf("pod failed to enter running state: %s", status)
return
}
exec := ExecOptions{
PodName: s.pod.Name,
Namespace: s.pod.Namespace,
ContainerName: name,
Command: command,
In: strings.NewReader(script),
Out: s.Trace,
Err: s.Trace,
Stdin: true,
Config: s.kubeConfig,
Client: s.kubeClient,
Executor: &DefaultRemoteExecutor{},
}
retryable := retry.New(retry.WithBuildLog(&exec, &s.BuildLogger))
errCh <- retryable.Run()
}()
return errCh
}
func (s *executor) prepareOverwrites(variables common.JobVariables) error {
values, err := createOverwrites(s.Config.Kubernetes, variables, s.BuildLogger)
if err != nil {
return err
}
s.configurationOverwrites = values
return nil
}
func (s *executor) prepareOptions(build *common.Build) {
s.options = &kubernetesOptions{}
s.options.Image = build.Image
s.getServices(build)
}
func (s *executor) getServices(build *common.Build) {
for _, service := range s.Config.Kubernetes.Services {
if service.Name == "" {
continue
}
s.options.Services = append(s.options.Services, service.ToImageDefinition())
}
for _, service := range build.Services {
if service.Name == "" {
continue
}
s.options.Services = append(s.options.Services, service)
}
}
// checkDefaults Defines the configuration for the Pod on Kubernetes
func (s *executor) checkDefaults() error {
if s.options.Image.Name == "" {
if s.Config.Kubernetes.Image == "" {
return fmt.Errorf("no image specified and no default set in config")
}
s.options.Image = common.Image{
Name: s.Config.Kubernetes.Image,
}
}
if s.configurationOverwrites.namespace == "" {
s.Warningln("Namespace is empty, therefore assuming 'default'.")
s.configurationOverwrites.namespace = "default"
}
s.Println("Using Kubernetes namespace:", s.configurationOverwrites.namespace)
return nil
}
func IsKubernetesPodNotFoundError(err error) bool {
var statusErr *kubeerrors.StatusError
return errors.As(err, &statusErr) &&
statusErr.ErrStatus.Code == http.StatusNotFound &&
statusErr.ErrStatus.Details != nil &&
statusErr.ErrStatus.Details.Kind == "pods"
}
func newExecutor() *executor {
e := &executor{
AbstractExecutor: executors.AbstractExecutor{
ExecutorOptions: executorOptions,
},
remoteProcessTerminated: make(chan shells.TrapCommandExitStatus),
}
e.newLogProcessor = func() logProcessor {
return newKubernetesLogProcessor(
e.kubeClient,
e.kubeConfig,
&backoff.Backoff{Min: time.Second, Max: 30 * time.Second},
e.Build.Log(),
kubernetesLogProcessorPodConfig{
namespace: e.pod.Namespace,
pod: e.pod.Name,
container: helperContainerName,
logPath: e.logFile(),
waitLogFileTimeout: waitLogFileTimeout,
},
)
}
return e
}
func featuresFn(features *common.FeaturesInfo) {
features.Variables = true
features.Image = true
features.Services = true
features.Artifacts = true
features.Cache = true
features.Session = true
features.Terminal = true
features.Proxy = true
}
func init() {
common.RegisterExecutorProvider("kubernetes", executors.DefaultExecutorProvider{
Creator: func() common.Executor {
return newExecutor()
},
FeaturesUpdater: featuresFn,
DefaultShellName: executorOptions.Shell.Shell,
})
}
package kubernetes
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"strconv"
"strings"
"time"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
)
type logStreamer interface {
Stream(offset int64, output io.Writer) error
fmt.Stringer
}
type kubernetesLogStreamer struct {
kubernetesLogProcessorPodConfig
client *kubernetes.Clientset
clientConfig *restclient.Config
executor RemoteExecutor
}
func (s *kubernetesLogStreamer) Stream(offset int64, output io.Writer) error {
exec := ExecOptions{
Namespace: s.namespace,
PodName: s.pod,
ContainerName: s.container,
Stdin: false,
Command: []string{
"gitlab-runner-helper",
"read-logs",
"--path",
s.logPath,
"--offset",
strconv.FormatInt(offset, 10),
"--wait-file-timeout",
s.waitLogFileTimeout.String(),
},
Out: output,
Err: output,
Executor: s.executor,
Client: s.client,
Config: s.clientConfig,
}
return exec.executeRequest()
}
func (s *kubernetesLogStreamer) String() string {
return fmt.Sprintf("%s/%s/%s:%s", s.namespace, s.pod, s.container, s.logPath)
}
type logProcessor interface {
// Process listens for log lines
// consumers must read from the channel until it's closed
// consumers are also notified in case of error through the error channel
Process(ctx context.Context) (<-chan string, <-chan error)
}
type backoffCalculator interface {
ForAttempt(attempt float64) time.Duration
}
// kubernetesLogProcessor processes the logs from a container and tries to reattach
// to the stream constantly, stopping only when the passed context is cancelled.
type kubernetesLogProcessor struct {
backoff backoffCalculator
logger logrus.FieldLogger
logStreamer logStreamer
logsOffset int64
}
type kubernetesLogProcessorPodConfig struct {
namespace string
pod string
container string
logPath string
waitLogFileTimeout time.Duration
}
func newKubernetesLogProcessor(
client *kubernetes.Clientset,
clientConfig *restclient.Config,
backoff backoffCalculator,
logger logrus.FieldLogger,
podCfg kubernetesLogProcessorPodConfig,
) *kubernetesLogProcessor {
logStreamer := &kubernetesLogStreamer{
kubernetesLogProcessorPodConfig: podCfg,
client: client,
clientConfig: clientConfig,
executor: new(DefaultRemoteExecutor),
}
return &kubernetesLogProcessor{
backoff: backoff,
logger: logger,
logStreamer: logStreamer,
}
}
func (l *kubernetesLogProcessor) Process(ctx context.Context) (<-chan string, <-chan error) {
outCh := make(chan string)
errCh := make(chan error)
go func() {
defer close(outCh)
defer close(errCh)
l.attach(ctx, outCh, errCh)
}()
return outCh, errCh
}
func (l *kubernetesLogProcessor) attach(ctx context.Context, outCh chan string, errCh chan error) {
var (
attempt float64 = -1
backoffDuration time.Duration
)
for {
// We do not exit because we need the processLogs goroutine still running.
// Once the error message is sent, a new step cleanup variables is started.
// As the pod is still running, the processLogs goroutine is not launched anymore.
// This is why, even though the error is sent to fail the ongoing step,
// we keep trying to reconnect to the output log, as a new one is created for variables cleanup.
attempt++
if attempt > 0 {
backoffDuration = l.backoff.ForAttempt(attempt)
l.logger.Debugln(fmt.Sprintf(
"Backing off reattaching log for %s for %s (attempt %f)",
l.logStreamer,
backoffDuration,
attempt,
))
}
select {
case <-ctx.Done():
l.logger.Debugln(fmt.Sprintf("Detaching from log... %v", ctx.Err()))
return
case <-time.After(backoffDuration):
err := l.processStream(ctx, outCh)
exitCode := getExitCode(err)
switch {
case exitCode == outputLogFileNotExistsExitCode:
// The cleanup variables step recreates a new output.log file
// where the shells.TrapCommandExitStatus is written.
// To not miss this line, we need to have the offset reset when we reconnect to the newly created log
l.logsOffset = 0
errCh <- fmt.Errorf("output log file deleted, cannot continue %w", err)
case err != nil:
l.logger.Warningln(fmt.Sprintf("Error %v. Retrying...", err))
default:
l.logger.Debug("processStream exited with no error")
}
}
}
}
func (l *kubernetesLogProcessor) processStream(ctx context.Context, outCh chan string) error {
reader, writer := io.Pipe()
defer func() {
_ = reader.Close()
_ = writer.Close()
}()
// Using errgroup.WithContext doesn't work here since if either one of the goroutines
// exits with a nil error, we can't signal the other one to exit
ctx, cancel := context.WithCancel(ctx)
var gr errgroup.Group
logsOffset := l.logsOffset
gr.Go(func() error {
defer cancel()
err := l.logStreamer.Stream(logsOffset, writer)
// prevent printing an error that the container exited
// when the context is already cancelled
if errors.Is(ctx.Err(), context.Canceled) {
return nil
}
if err != nil {
err = fmt.Errorf("streaming logs %s: %w", l.logStreamer, err)
}
return err
})
gr.Go(func() error {
defer cancel()
err := l.readLogs(ctx, reader, outCh)
if err != nil {
err = fmt.Errorf("reading logs %s: %w", l.logStreamer, err)
}
return err
})
return gr.Wait()
}
func (l *kubernetesLogProcessor) readLogs(ctx context.Context, logs io.Reader, outCh chan string) error {
logsScanner, linesCh := l.scan(ctx, logs)
for {
select {
case <-ctx.Done():
return nil
case line, more := <-linesCh:
if !more {
l.logger.Debug("No more data in linesCh")
return logsScanner.Err()
}
newLogsOffset, logLine := l.parseLogLine(line)
if newLogsOffset != -1 {
l.logsOffset = newLogsOffset
}
outCh <- logLine
}
}
}
func (l *kubernetesLogProcessor) scan(ctx context.Context, logs io.Reader) (*bufio.Scanner, <-chan string) {
logsScanner := bufio.NewScanner(logs)
linesCh := make(chan string)
go func() {
defer close(linesCh)
// This goroutine will exit when the calling method closes the logs stream or the context is cancelled
for logsScanner.Scan() {
select {
case <-ctx.Done():
return
case linesCh <- logsScanner.Text():
}
}
}()
return logsScanner, linesCh
}
// Each line starts with its bytes offset. We need this to resume the log from that point
// if we detach for some reason. The format is "10 log line continues as normal".
// The line doesn't include the new line character.
// Lines without offset are acceptable and return -1 for offset.
func (l *kubernetesLogProcessor) parseLogLine(line string) (int64, string) {
if line == "" {
return -1, ""
}
offsetIndex := strings.Index(line, " ")
if offsetIndex == -1 {
return -1, line
}
offset := line[:offsetIndex]
parsedOffset, err := strconv.ParseInt(offset, 10, 64)
if err != nil {
return -1, line
}
logLine := line[offsetIndex+1:]
return parsedOffset, logLine
}
package kubernetes
import (
"fmt"
"regexp"
"strings"
api "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
const (
// NamespaceOverwriteVariableName is the key for the JobVariable containing user overwritten Namespace
NamespaceOverwriteVariableName = "KUBERNETES_NAMESPACE_OVERWRITE"
// ServiceAccountOverwriteVariableName is the key for the JobVariable containing user overwritten ServiceAccount
ServiceAccountOverwriteVariableName = "KUBERNETES_SERVICE_ACCOUNT_OVERWRITE"
// BearerTokenOverwriteVariableValue is the key for the JobVariable containing user overwritten BearerToken
BearerTokenOverwriteVariableValue = "KUBERNETES_BEARER_TOKEN"
// PodAnnotationsOverwriteVariablePrefix is the prefix for all the JobVariable keys containing
// user overwritten PodAnnotations
PodAnnotationsOverwriteVariablePrefix = "KUBERNETES_POD_ANNOTATIONS_"
// CPULimitOverwriteVariableValue is the key for the JobVariable containing user overwritten cpu limit
CPULimitOverwriteVariableValue = "KUBERNETES_CPU_LIMIT"
// CPURequestOverwriteVariableValue is the key for the JobVariable containing user overwritten cpu limit
CPURequestOverwriteVariableValue = "KUBERNETES_CPU_REQUEST"
// MemoryLimitOverwriteVariableValue is the key for the JobVariable containing user overwritten memory limit
MemoryLimitOverwriteVariableValue = "KUBERNETES_MEMORY_LIMIT"
// MemoryRequestOverwriteVariableValue is the key for the JobVariable containing user overwritten memory limit
MemoryRequestOverwriteVariableValue = "KUBERNETES_MEMORY_REQUEST"
// EphemeralStorageLimitOverwriteVariableValue is the key for the JobVariable containing user overwritten
// ephemeral storage limit
EphemeralStorageLimitOverwriteVariableValue = "KUBERNETES_EPHEMERAL_STORAGE_LIMIT"
// EphemeralStorageRequestOverwriteVariableValue is the key for the JobVariable containing user overwritten
// ephemeral storage limit
EphemeralStorageRequestOverwriteVariableValue = "KUBERNETES_EPHEMERAL_STORAGE_REQUEST"
// ServiceCPULimitOverwriteVariableValue is the key for the JobVariable containing user overwritten service cpu
// limit
ServiceCPULimitOverwriteVariableValue = "KUBERNETES_SERVICE_CPU_LIMIT"
// ServiceCPURequestOverwriteVariableValue is the key for the JobVariable containing user overwritten service cpu
// limit
ServiceCPURequestOverwriteVariableValue = "KUBERNETES_SERVICE_CPU_REQUEST"
// ServiceMemoryLimitOverwriteVariableValue is the key for the JobVariable containing user overwritten service
// memory limit
ServiceMemoryLimitOverwriteVariableValue = "KUBERNETES_SERVICE_MEMORY_LIMIT"
// ServiceMemoryRequestOverwriteVariableValue is the key for the JobVariable containing user overwritten service
// memory limit
ServiceMemoryRequestOverwriteVariableValue = "KUBERNETES_SERVICE_MEMORY_REQUEST"
// ServiceEphemeralStorageLimitOverwriteVariableValue is the key for the JobVariable containing user overwritten
// service ephemeral storage
ServiceEphemeralStorageLimitOverwriteVariableValue = "KUBERNETES_SERVICE_EPHEMERAL_STORAGE_LIMIT"
// ServiceEphemeralStorageRequestOverwriteVariableValue is the key for the JobVariable containing user overwritten
// service ephemeral storage
ServiceEphemeralStorageRequestOverwriteVariableValue = "KUBERNETES_SERVICE_EPHEMERAL_STORAGE_REQUEST"
// HelperCPULimitOverwriteVariableValue is the key for the JobVariable containing user overwritten helper cpu limit
HelperCPULimitOverwriteVariableValue = "KUBERNETES_HELPER_CPU_LIMIT"
// HelperCPURequestOverwriteVariableValue is the key for the JobVariable containing user overwritten helper cpu
// limit
HelperCPURequestOverwriteVariableValue = "KUBERNETES_HELPER_CPU_REQUEST"
// HelperMemoryLimitOverwriteVariableValue is the key for the JobVariable containing user overwritten helper memory
// limit
HelperMemoryLimitOverwriteVariableValue = "KUBERNETES_HELPER_MEMORY_LIMIT"
// HelperMemoryRequestOverwriteVariableValue is the key for the JobVariable containing user overwritten helper
// memory limit
HelperEphemeralStorageRequestOverwriteVariableValue = "KUBERNETES_HELPER_EPHEMERAL_STORAGE_REQUEST"
// HelperEphemeralStorageLimitOverwriteVariableValue is the key for the JobVariable containing user overwritten
// helper ephemeral storage
HelperEphemeralStorageLimitOverwriteVariableValue = "KUBERNETES_HELPER_EPHEMERAL_STORAGE_LIMIT"
// HelperEphemeralStorageRequestOverwriteVariableValue is the key for the JobVariable containing user overwritten
// ephemeral storage
HelperMemoryRequestOverwriteVariableValue = "KUBERNETES_HELPER_MEMORY_REQUEST"
)
type overwriteTooHighError struct {
resource string
max string
overwrite string
}
func (o *overwriteTooHighError) Error() string {
return fmt.Sprintf("the resource %q requested %q is higher than limit allowed %q", o.resource, o.overwrite, o.max)
}
func (o *overwriteTooHighError) Is(err error) bool {
_, ok := err.(*overwriteTooHighError)
return ok
}
type malformedOverwriteError struct {
value string
pattern string
}
func (m *malformedOverwriteError) Error() string {
return fmt.Sprintf("provided value %q does not match %q", m.value, m.pattern)
}
func (m *malformedOverwriteError) Is(err error) bool {
_, ok := err.(*malformedOverwriteError)
return ok
}
type overwrites struct {
namespace string
serviceAccount string
bearerToken string
podAnnotations map[string]string
buildLimits api.ResourceList
serviceLimits api.ResourceList
helperLimits api.ResourceList
buildRequests api.ResourceList
serviceRequests api.ResourceList
helperRequests api.ResourceList
}
//nolint:funlen
func createOverwrites(
config *common.KubernetesConfig,
variables common.JobVariables,
logger common.BuildLogger,
) (*overwrites, error) {
var err error
o := &overwrites{}
variables = variables.Expand()
namespaceOverwrite := variables.Get(NamespaceOverwriteVariableName)
o.namespace, err = o.evaluateOverwrite(
"Namespace",
config.Namespace,
config.NamespaceOverwriteAllowed,
namespaceOverwrite,
logger,
)
if err != nil {
return nil, err
}
serviceAccountOverwrite := variables.Get(ServiceAccountOverwriteVariableName)
o.serviceAccount, err = o.evaluateOverwrite(
"ServiceAccount",
config.ServiceAccount,
config.ServiceAccountOverwriteAllowed,
serviceAccountOverwrite,
logger,
)
if err != nil {
return nil, err
}
bearerTokenOverwrite := variables.Get(BearerTokenOverwriteVariableValue)
o.bearerToken, err = o.evaluateBoolControlledOverwrite(
"BearerToken",
config.BearerToken,
config.BearerTokenOverwriteAllowed,
bearerTokenOverwrite,
logger,
)
if err != nil {
return nil, err
}
o.podAnnotations, err = o.evaluateMapOverwrite(
"PodAnnotations",
config.PodAnnotations,
config.PodAnnotationsOverwriteAllowed,
variables,
PodAnnotationsOverwriteVariablePrefix,
logger,
)
if err != nil {
return nil, err
}
err = o.evaluateMaxBuildResourcesOverwrite(config, variables, logger)
if err != nil {
return nil, err
}
err = o.evaluateMaxServiceResourcesOverwrite(config, variables, logger)
if err != nil {
return nil, err
}
err = o.evaluateMaxHelperResourcesOverwrite(config, variables, logger)
if err != nil {
return nil, err
}
return o, nil
}
func (o *overwrites) evaluateMaxBuildResourcesOverwrite(
config *common.KubernetesConfig,
variables common.JobVariables,
logger common.BuildLogger,
) (err error) {
o.buildRequests, err = o.evaluateMaxResourceListOverwrite(
"CPURequest",
"MemoryRequest",
"EphemeralStorageRequest",
config.CPURequest,
config.MemoryRequest,
config.EphemeralStorageRequest,
config.CPURequestOverwriteMaxAllowed,
config.MemoryRequestOverwriteMaxAllowed,
config.EphemeralStorageRequestOverwriteMaxAllowed,
variables.Get(CPURequestOverwriteVariableValue),
variables.Get(MemoryRequestOverwriteVariableValue),
variables.Get(EphemeralStorageRequestOverwriteVariableValue),
logger,
)
if err != nil {
return fmt.Errorf("invalid build requests specified: %w", err)
}
o.buildLimits, err = o.evaluateMaxResourceListOverwrite(
"CPULimit",
"MemoryLimit",
"EphemeralStorageLimit",
config.CPULimit,
config.MemoryLimit,
config.EphemeralStorageLimit,
config.CPULimitOverwriteMaxAllowed,
config.MemoryLimitOverwriteMaxAllowed,
config.EphemeralStorageLimitOverwriteMaxAllowed,
variables.Get(CPULimitOverwriteVariableValue),
variables.Get(MemoryLimitOverwriteVariableValue),
variables.Get(EphemeralStorageLimitOverwriteVariableValue),
logger,
)
if err != nil {
return fmt.Errorf("invalid build limits specified: %w", err)
}
return nil
}
func (o *overwrites) evaluateMaxServiceResourcesOverwrite(
config *common.KubernetesConfig,
variables common.JobVariables,
logger common.BuildLogger,
) (err error) {
o.serviceRequests, err = o.evaluateMaxResourceListOverwrite(
"ServiceCPURequest",
"ServiceMemoryRequest",
"ServiceEphemeralStorageRequest",
config.ServiceCPURequest,
config.ServiceMemoryRequest,
config.ServiceEphemeralStorageRequest,
config.ServiceCPURequestOverwriteMaxAllowed,
config.ServiceMemoryRequestOverwriteMaxAllowed,
config.ServiceEphemeralStorageRequestOverwriteMaxAllowed,
variables.Get(ServiceCPURequestOverwriteVariableValue),
variables.Get(ServiceMemoryRequestOverwriteVariableValue),
variables.Get(ServiceEphemeralStorageRequestOverwriteVariableValue),
logger,
)
if err != nil {
return fmt.Errorf("invalid service requests specified: %w", err)
}
o.serviceLimits, err = o.evaluateMaxResourceListOverwrite(
"ServiceCPULimit",
"ServiceMemoryLimit",
"ServiceEphemeralStorageLimit",
config.ServiceCPULimit,
config.ServiceMemoryLimit,
config.ServiceEphemeralStorageLimit,
config.ServiceCPULimitOverwriteMaxAllowed,
config.ServiceMemoryLimitOverwriteMaxAllowed,
config.ServiceEphemeralStorageLimitOverwriteMaxAllowed,
variables.Get(ServiceCPULimitOverwriteVariableValue),
variables.Get(ServiceMemoryLimitOverwriteVariableValue),
variables.Get(ServiceEphemeralStorageLimitOverwriteVariableValue),
logger,
)
if err != nil {
return fmt.Errorf("invalid service limits specified: %w", err)
}
return nil
}
func (o *overwrites) evaluateMaxHelperResourcesOverwrite(
config *common.KubernetesConfig,
variables common.JobVariables,
logger common.BuildLogger,
) (err error) {
o.helperRequests, err = o.evaluateMaxResourceListOverwrite(
"HelperCPURequest",
"HelperMemoryRequest",
"HelperEphemeralStorageRequest",
config.HelperCPURequest,
config.HelperMemoryRequest,
config.HelperEphemeralStorageRequest,
config.HelperCPURequestOverwriteMaxAllowed,
config.HelperMemoryRequestOverwriteMaxAllowed,
config.HelperEphemeralStorageRequestOverwriteMaxAllowed,
variables.Get(HelperCPURequestOverwriteVariableValue),
variables.Get(HelperMemoryRequestOverwriteVariableValue),
variables.Get(HelperEphemeralStorageRequestOverwriteVariableValue),
logger,
)
if err != nil {
return fmt.Errorf("invalid helper requests specified: %w", err)
}
o.helperLimits, err = o.evaluateMaxResourceListOverwrite(
"HelperCPULimit",
"HelperMemoryLimit",
"HelperEphemeralStorageLimit",
config.HelperCPULimit,
config.HelperMemoryLimit,
config.HelperEphemeralStorageLimit,
config.HelperCPULimitOverwriteMaxAllowed,
config.HelperMemoryLimitOverwriteMaxAllowed,
config.HelperEphemeralStorageLimitOverwriteMaxAllowed,
variables.Get(HelperCPULimitOverwriteVariableValue),
variables.Get(HelperMemoryLimitOverwriteVariableValue),
variables.Get(HelperEphemeralStorageLimitOverwriteVariableValue),
logger,
)
if err != nil {
return fmt.Errorf("invalid helper limits specified: %w", err)
}
return nil
}
func (o *overwrites) evaluateBoolControlledOverwrite(
fieldName, value string,
canOverride bool,
overwriteValue string,
logger common.BuildLogger,
) (string, error) {
if canOverride {
return o.evaluateOverwrite(fieldName, value, ".+", overwriteValue, logger)
}
return o.evaluateOverwrite(fieldName, value, "", overwriteValue, logger)
}
func (o *overwrites) evaluateOverwrite(
fieldName, value, regex, overwriteValue string,
logger common.BuildLogger,
) (string, error) {
if regex == "" {
logger.Debugln("Regex allowing overrides for", fieldName, "is empty, disabling override.")
return value, nil
}
if overwriteValue == "" {
return value, nil
}
if err := overwriteRegexCheck(regex, overwriteValue); err != nil {
return value, err
}
logValue := overwriteValue
if fieldName == "BearerToken" {
logValue = "XXXXXXXX..."
}
logger.Println(fmt.Sprintf("%q overwritten with %q", fieldName, logValue))
return overwriteValue, nil
}
func overwriteRegexCheck(regex, value string) error {
var err error
var r *regexp.Regexp
if r, err = regexp.Compile(regex); err != nil {
return err
}
if match := r.MatchString(value); !match {
return &malformedOverwriteError{value: value, pattern: regex}
}
return nil
}
// splitMapOverwrite splits provided string on the first "=" and returns (key, value, nil).
// If the argument cannot be split an error is returned
func splitMapOverwrite(str string) (string, string, error) {
if split := strings.SplitN(str, "=", 2); len(split) > 1 {
return split[0], split[1], nil
}
return "", "", &malformedOverwriteError{value: str, pattern: "k=v"}
}
func (o *overwrites) evaluateMapOverwrite(
fieldName string,
values map[string]string,
regex string,
variables common.JobVariables,
variablesSelector string,
logger common.BuildLogger,
) (map[string]string, error) {
if regex == "" {
logger.Debugln("Regex allowing overrides for", fieldName, "is empty, disabling override.")
return values, nil
}
finalValues := make(map[string]string)
for k, v := range values {
finalValues[k] = v
}
for _, variable := range variables {
if !strings.HasPrefix(variable.Key, variablesSelector) {
continue
}
if err := overwriteRegexCheck(regex, variable.Value); err != nil {
return nil, err
}
key, value, err := splitMapOverwrite(variable.Value)
if err != nil {
return nil, err
}
finalValues[key] = value
logger.Println(fmt.Sprintf("%q %q overwritten with %q", fieldName, key, value))
}
return finalValues, nil
}
func (o *overwrites) evaluateMaxResourceListOverwrite(
cpuFieldName,
memoryFieldName,
ephemeralStorageFieldName,
currentCPU,
currentMemory,
currentEphemeralStorage,
maxCPU,
maxMemory,
maxEphemeralStorage,
overwriteCPU,
overwriteMemory string,
overwriteEphemeralStorage string,
logger common.BuildLogger,
) (api.ResourceList, error) {
cpu, err := o.evaluateMaxResourceOverwrite(cpuFieldName, currentCPU, maxCPU, overwriteCPU, logger)
if err != nil {
return nil, err
}
memory, err := o.evaluateMaxResourceOverwrite(memoryFieldName, currentMemory, maxMemory, overwriteMemory, logger)
if err != nil {
return nil, err
}
ephemeralStorage, err := o.evaluateMaxResourceOverwrite(
ephemeralStorageFieldName,
currentEphemeralStorage,
maxEphemeralStorage,
overwriteEphemeralStorage,
logger,
)
if err != nil {
return nil, err
}
return createResourceList(cpu, memory, ephemeralStorage)
}
func (o *overwrites) evaluateMaxResourceOverwrite(
fieldName,
value,
maxResource,
overwriteValue string,
logger common.BuildLogger,
) (string, error) {
if maxResource == "" {
logger.Debugln("setting allowing overrides for", fieldName, "is empty, disabling override.")
return value, nil
}
if overwriteValue == "" {
return value, nil
}
var rMaxResource, rOverwriteValue resource.Quantity
var err error
if rMaxResource, err = resource.ParseQuantity(maxResource); err != nil {
return value, fmt.Errorf("parsing resource limit: %q", err.Error())
}
if rOverwriteValue, err = resource.ParseQuantity(overwriteValue); err != nil {
return value, fmt.Errorf("parsing resource limit: %q", err.Error())
}
cmp := rOverwriteValue.Cmp(rMaxResource)
if cmp == 1 {
return "", &overwriteTooHighError{
resource: fieldName,
max: maxResource,
overwrite: overwriteValue,
}
}
logger.Println(fmt.Sprintf("%q overwritten with %q", fieldName, overwriteValue))
return overwriteValue, nil
}
package kubernetes
import (
"context"
"fmt"
"io"
"net/http"
"strconv"
"github.com/gorilla/websocket"
"github.com/sirupsen/logrus"
terminal "gitlab.com/gitlab-org/gitlab-terminal"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8net "k8s.io/apimachinery/pkg/util/net"
"k8s.io/client-go/rest"
"gitlab.com/gitlab-org/gitlab-runner/session/proxy"
)
const runningState = "Running"
func (s *executor) Pool() proxy.Pool {
return s.ProxyPool
}
func (s *executor) newProxy(serviceName string, ports []proxy.Port) *proxy.Proxy {
return &proxy.Proxy{
Settings: proxy.NewProxySettings(serviceName, ports),
ConnectionHandler: s,
}
}
func (s *executor) ProxyRequest(
w http.ResponseWriter,
r *http.Request,
requestedURI string,
port string,
settings *proxy.Settings,
) {
logger := logrus.WithFields(logrus.Fields{
"uri": r.RequestURI,
"method": r.Method,
"port": port,
"settings": settings,
})
portSettings, err := settings.PortByNameOrNumber(port)
if err != nil {
logger.WithError(err).Errorf("port proxy %q not found", port)
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
if !s.servicesRunning() {
logger.Errorf("services are not ready yet")
http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
return
}
if websocket.IsWebSocketUpgrade(r) {
proxyWSRequest(s, w, r, requestedURI, portSettings, settings, logger)
return
}
proxyHTTPRequest(s, w, r, requestedURI, portSettings, settings, logger)
}
func (s *executor) servicesRunning() bool {
// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
pod, err := s.kubeClient.CoreV1().Pods(s.pod.Namespace).Get(context.TODO(), s.pod.Name, metav1.GetOptions{})
if err != nil || pod.Status.Phase != runningState {
return false
}
for _, container := range pod.Status.ContainerStatuses {
if !container.Ready {
return false
}
}
return true
}
func (s *executor) serviceEndpointRequest(
verb, serviceName, requestedURI string,
port proxy.Port,
) (*rest.Request, error) {
scheme, err := port.Scheme()
if err != nil {
return nil, err
}
result := s.kubeClient.CoreV1().RESTClient().Verb(verb).
Namespace(s.pod.Namespace).
Resource("services").
SubResource("proxy").
Name(k8net.JoinSchemeNamePort(scheme, serviceName, strconv.Itoa(port.Number))).
Suffix(requestedURI)
return result, nil
}
func proxyWSRequest(
s *executor,
w http.ResponseWriter,
r *http.Request,
requestedURI string,
port proxy.Port,
proxySettings *proxy.Settings,
logger *logrus.Entry,
) {
// In order to avoid calling this method, and use one of its own,
// we should refactor the library "gitlab.com/gitlab-org/gitlab-terminal"
// and make it more generic, not so terminal focused, with a broader
// terminology. (https://gitlab.com/gitlab-org/gitlab-runner/issues/4059)
settings, err := s.getTerminalSettings()
if err != nil {
logger.WithError(err).Errorf("service proxy: error getting WS settings")
http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
return
}
req, err := s.serviceEndpointRequest(r.Method, proxySettings.ServiceName, requestedURI, port)
if err != nil {
logger.WithError(err).Errorf("service proxy: error proxying WS request")
http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
return
}
u := req.URL()
u.Scheme = proxy.WebsocketProtocolFor(u.Scheme)
settings.Url = u.String()
serviceProxy := terminal.NewWebSocketProxy(1)
terminal.ProxyWebSocket(w, r, settings, serviceProxy)
}
func proxyHTTPRequest(
s *executor,
w http.ResponseWriter,
r *http.Request,
requestedURI string,
port proxy.Port,
proxy *proxy.Settings,
logger *logrus.Entry,
) {
req, err := s.serviceEndpointRequest(r.Method, proxy.ServiceName, requestedURI, port)
if err != nil {
logger.WithError(err).Errorf("service proxy: error proxying HTTP request")
http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
return
}
// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
body, err := req.Stream(context.TODO())
if err != nil {
message, code := handleProxyHTTPErr(err, logger)
w.WriteHeader(code)
if message != "" {
_, _ = fmt.Fprint(w, message)
}
return
}
w.WriteHeader(http.StatusOK)
_, _ = io.Copy(w, body)
}
func handleProxyHTTPErr(err error, logger *logrus.Entry) (string, int) {
statusError, ok := err.(*errors.StatusError)
if !ok {
return "", http.StatusInternalServerError
}
code := int(statusError.Status().Code)
// When the error is a 503 we don't want to give any information
// coming from Kubernetes
if code == http.StatusServiceUnavailable {
logger.Error(statusError.Status().Message)
return "", code
}
details := statusError.Status().Details
if details == nil {
return "", code
}
causes := details.Causes
if len(causes) > 0 {
return causes[0].Message, code
}
return "", code
}
package kubernetes
import (
"io/ioutil"
"net/http"
"net/url"
"gitlab.com/gitlab-org/gitlab-runner/session/proxy"
terminalsession "gitlab.com/gitlab-org/gitlab-runner/session/terminal"
terminal "gitlab.com/gitlab-org/gitlab-terminal"
api "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes/scheme"
)
func (s *executor) Connect() (terminalsession.Conn, error) {
settings, err := s.getTerminalSettings()
if err != nil {
return nil, err
}
return terminalConn{settings: settings}, nil
}
type terminalConn struct {
settings *terminal.TerminalSettings
}
func (t terminalConn) Start(w http.ResponseWriter, r *http.Request, timeoutCh, disconnectCh chan error) {
wsProxy := terminal.NewWebSocketProxy(1) // one stopper: terminal exit handler
terminalsession.ProxyTerminal(
timeoutCh,
disconnectCh,
wsProxy.StopCh,
func() {
terminal.ProxyWebSocket(w, r, t.settings, wsProxy)
},
)
}
func (t terminalConn) Close() error {
return nil
}
func (s *executor) getTerminalSettings() (*terminal.TerminalSettings, error) {
config, err := getKubeClientConfig(s.Config.Kubernetes, s.configurationOverwrites)
if err != nil {
return nil, err
}
wsURL := s.getTerminalWebSocketURL()
if err != nil {
return nil, err
}
caCert := ""
if len(config.CAFile) > 0 {
buf, err := ioutil.ReadFile(config.CAFile)
if err != nil {
return nil, err
}
caCert = string(buf)
}
term := &terminal.TerminalSettings{
Subprotocols: []string{"channel.k8s.io"},
Url: wsURL.String(),
Header: http.Header{"Authorization": []string{"Bearer " + config.BearerToken}},
CAPem: caCert,
MaxSessionTime: 0,
}
return term, nil
}
func (s *executor) getTerminalWebSocketURL() *url.URL {
wsURL := s.kubeClient.CoreV1().RESTClient().Post().
Namespace(s.pod.Namespace).
Resource("pods").
Name(s.pod.Name).
SubResource("exec").
VersionedParams(&api.PodExecOptions{
Stdin: true,
Stdout: true,
Stderr: true,
TTY: true,
Container: "build",
Command: []string{"sh", "-c", "bash || sh"},
}, scheme.ParameterCodec).URL()
wsURL.Scheme = proxy.WebsocketProtocolFor(wsURL.Scheme)
return wsURL
}
package kubernetes
import (
"errors"
"fmt"
"io"
"net/http"
"time"
"golang.org/x/net/context"
api "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors/kubernetes/internal/pull"
)
type kubeConfigProvider func() (*restclient.Config, error)
type resourceQuantityError struct {
resource string
value string
inner error
}
func (r *resourceQuantityError) Error() string {
return fmt.Sprintf("parsing resource %q with value %q: %q", r.resource, r.value, r.inner)
}
func (r *resourceQuantityError) Is(err error) bool {
t, ok := err.(*resourceQuantityError)
return ok && r.resource == t.resource && r.value == t.value && r.inner == t.inner
}
var (
// inClusterConfig parses kubernetes configuration reading in cluster values
inClusterConfig kubeConfigProvider = restclient.InClusterConfig
// defaultKubectlConfig parses kubectl configuration ad loads the default cluster
defaultKubectlConfig kubeConfigProvider = loadDefaultKubectlConfig
)
func getKubeClientConfig(
config *common.KubernetesConfig,
overwrites *overwrites,
) (kubeConfig *restclient.Config, err error) {
if len(config.Host) > 0 {
kubeConfig, err = getOutClusterClientConfig(config)
} else {
kubeConfig, err = guessClientConfig()
}
if err != nil {
return nil, err
}
// apply overwrites
if overwrites.bearerToken != "" {
kubeConfig.BearerToken = overwrites.bearerToken
}
kubeConfig.UserAgent = common.AppVersion.UserAgent()
return kubeConfig, nil
}
func getOutClusterClientConfig(config *common.KubernetesConfig) (*restclient.Config, error) {
kubeConfig := &restclient.Config{
Host: config.Host,
BearerToken: config.BearerToken,
TLSClientConfig: restclient.TLSClientConfig{
CAFile: config.CAFile,
},
}
// certificate based auth
if config.CertFile != "" {
if config.KeyFile == "" || config.CAFile == "" {
return nil, fmt.Errorf("ca file, cert file and key file must be specified when using file based auth")
}
kubeConfig.TLSClientConfig.CertFile = config.CertFile
kubeConfig.TLSClientConfig.KeyFile = config.KeyFile
}
return kubeConfig, nil
}
func guessClientConfig() (*restclient.Config, error) {
// Try in cluster config first
if inClusterCfg, err := inClusterConfig(); err == nil {
return inClusterCfg, nil
}
// in cluster config failed. Reading default kubectl config
return defaultKubectlConfig()
}
func loadDefaultKubectlConfig() (*restclient.Config, error) {
config, err := clientcmd.NewDefaultClientConfigLoadingRules().Load()
if err != nil {
return nil, err
}
return clientcmd.NewDefaultClientConfig(*config, &clientcmd.ConfigOverrides{}).ClientConfig()
}
func closeKubeClient(client *kubernetes.Clientset) bool {
if client == nil {
return false
}
rest, ok := client.CoreV1().RESTClient().(*restclient.RESTClient)
if !ok || rest.Client == nil || rest.Client.Transport == nil {
return false
}
if transport, ok := rest.Client.Transport.(*http.Transport); ok {
transport.CloseIdleConnections()
return true
}
return false
}
func isRunning(pod *api.Pod) (bool, error) {
switch pod.Status.Phase {
case api.PodRunning:
return true, nil
case api.PodSucceeded:
return false, fmt.Errorf("pod already succeeded before it begins running")
case api.PodFailed:
return false, fmt.Errorf("pod status is failed")
default:
return false, nil
}
}
type podPhaseResponse struct {
done bool
phase api.PodPhase
err error
}
func getPodPhase(c *kubernetes.Clientset, pod *api.Pod, out io.Writer) podPhaseResponse {
// TODO: handle the context properly with https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27932
pod, err := c.CoreV1().Pods(pod.Namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{})
if err != nil {
return podPhaseResponse{true, api.PodUnknown, err}
}
ready, err := isRunning(pod)
if err != nil || ready {
return podPhaseResponse{true, pod.Status.Phase, err}
}
// check status of containers
for _, container := range append(pod.Status.ContainerStatuses, pod.Status.InitContainerStatuses...) {
if container.Ready {
continue
}
waiting := container.State.Waiting
if waiting == nil {
continue
}
switch waiting.Reason {
case "InvalidImageName":
err = &common.BuildError{Inner: fmt.Errorf("image pull failed: %s", waiting.Message)}
return podPhaseResponse{true, api.PodUnknown, err}
case "ErrImagePull", "ImagePullBackOff":
msg := fmt.Sprintf("image pull failed: %s", waiting.Message)
imagePullErr := &pull.ImagePullError{Message: msg, Image: container.Image}
return podPhaseResponse{
true,
api.PodUnknown,
&common.BuildError{Inner: imagePullErr, FailureReason: common.ScriptFailure},
}
}
}
_, _ = fmt.Fprintf(
out,
"Waiting for pod %s/%s to be running, status is %s\n",
pod.Namespace,
pod.Name,
pod.Status.Phase,
)
for _, condition := range pod.Status.Conditions {
// skip conditions with no reason, these are typically expected pod conditions
if condition.Reason == "" {
continue
}
_, _ = fmt.Fprintf(
out,
"\t%s: %q\n",
condition.Reason,
condition.Message,
)
}
return podPhaseResponse{false, pod.Status.Phase, nil}
}
func triggerPodPhaseCheck(c *kubernetes.Clientset, pod *api.Pod, out io.Writer) <-chan podPhaseResponse {
errc := make(chan podPhaseResponse)
go func() {
defer close(errc)
errc <- getPodPhase(c, pod, out)
}()
return errc
}
// waitForPodRunning will use client c to detect when pod reaches the PodRunning
// state. It returns the final PodPhase once either PodRunning, PodSucceeded or
// PodFailed has been reached. In the case of PodRunning, it will also wait until
// all containers within the pod are also Ready.
// It returns error if the call to retrieve pod details fails or the timeout is
// reached.
// The timeout and polling values are configurable through KubernetesConfig
// parameters.
func waitForPodRunning(
ctx context.Context,
c *kubernetes.Clientset,
pod *api.Pod,
out io.Writer,
config *common.KubernetesConfig,
) (api.PodPhase, error) {
pollInterval := config.GetPollInterval()
pollAttempts := config.GetPollAttempts()
for i := 0; i <= pollAttempts; i++ {
select {
case r := <-triggerPodPhaseCheck(c, pod, out):
if !r.done {
time.Sleep(time.Duration(pollInterval) * time.Second)
continue
}
return r.phase, r.err
case <-ctx.Done():
return api.PodUnknown, ctx.Err()
}
}
return api.PodUnknown, errors.New("timed out waiting for pod to start")
}
// limits takes a string representing CPU, memory and ephemeralStorage limits,
// and returns a ResourceList with appropriately scaled Quantity
// values for Kubernetes. This allows users to write "500m" for CPU,
// "50Mi" for memory and "1Gi" for ephemeral storage (etc.)
func createResourceList(cpu, memory, ephemeralStorage string) (api.ResourceList, error) {
var rCPU, rMem, rStor resource.Quantity
var err error
parse := func(s string) (resource.Quantity, error) {
var q resource.Quantity
if s == "" {
return q, nil
}
if q, err = resource.ParseQuantity(s); err != nil {
return q, err
}
return q, nil
}
if rCPU, err = parse(cpu); err != nil {
return api.ResourceList{}, &resourceQuantityError{resource: "cpu", value: cpu, inner: err}
}
if rMem, err = parse(memory); err != nil {
return api.ResourceList{}, &resourceQuantityError{resource: "memory", value: memory, inner: err}
}
if rStor, err = parse(ephemeralStorage); err != nil {
return api.ResourceList{}, &resourceQuantityError{
resource: "ephemeralStorage",
value: ephemeralStorage,
inner: err,
}
}
l := make(api.ResourceList)
q := resource.Quantity{}
if rCPU != q {
l[api.ResourceCPU] = rCPU
}
if rMem != q {
l[api.ResourceMemory] = rMem
}
if rStor != q {
l[api.ResourceEphemeralStorage] = rStor
}
return l, nil
}
// buildVariables converts a common.BuildVariables into a list of
// kubernetes EnvVar objects
func buildVariables(bv common.JobVariables) []api.EnvVar {
e := make([]api.EnvVar, len(bv))
for i, b := range bv {
e[i] = api.EnvVar{
Name: b.Key,
Value: b.Value,
}
}
return e
}
func getCapabilities(defaultCapDrop []string, capAdd []string, capDrop []string) *api.Capabilities {
enabled := make(map[string]bool)
for _, v := range defaultCapDrop {
enabled[v] = false
}
for _, v := range capAdd {
enabled[v] = true
}
for _, v := range capDrop {
enabled[v] = false
}
if len(enabled) < 1 {
return nil
}
return buildCapabilities(enabled)
}
func buildCapabilities(enabled map[string]bool) *api.Capabilities {
capabilities := new(api.Capabilities)
for c, add := range enabled {
if add {
capabilities.Add = append(capabilities.Add, api.Capability(c))
continue
}
capabilities.Drop = append(capabilities.Drop, api.Capability(c))
}
return capabilities
}
package parallels
import (
"errors"
"fmt"
"time"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors"
prl "gitlab.com/gitlab-org/gitlab-runner/helpers/parallels"
"gitlab.com/gitlab-org/gitlab-runner/helpers/ssh"
)
type executor struct {
executors.AbstractExecutor
vmName string
sshCommand ssh.Client
provisioned bool
ipAddress string
machineVerified bool
}
func (s *executor) waitForIPAddress(vmName string, seconds int) (string, error) {
var lastError error
if s.ipAddress != "" {
return s.ipAddress, nil
}
s.Debugln("Looking for MAC address...")
macAddr, err := prl.Mac(vmName)
if err != nil {
return "", err
}
s.Debugln("Requesting IP address...")
for i := 0; i < seconds; i++ {
ipAddr, err := prl.IPAddress(macAddr)
if err == nil {
s.Debugln("IP address found", ipAddr, "...")
s.ipAddress = ipAddr
return ipAddr, nil
}
lastError = err
time.Sleep(time.Second)
}
return "", lastError
}
func (s *executor) verifyMachine(vmName string) error {
if s.machineVerified {
return nil
}
ipAddr, err := s.waitForIPAddress(vmName, 120)
if err != nil {
return err
}
// Create SSH command
sshCommand := ssh.Client{
Config: *s.Config.SSH,
Stdout: s.Trace,
Stderr: s.Trace,
ConnectRetries: 30,
}
sshCommand.Host = ipAddr
s.Debugln("Connecting to SSH...")
err = sshCommand.Connect()
if err != nil {
return err
}
defer sshCommand.Cleanup()
err = sshCommand.Run(s.Context, ssh.Command{Command: []string{"exit"}})
if err != nil {
return err
}
s.machineVerified = true
return nil
}
func (s *executor) restoreFromSnapshot() error {
s.Debugln("Requesting default snapshot for VM...")
snapshot, err := prl.GetDefaultSnapshot(s.vmName)
if err != nil {
return err
}
s.Debugln("Reverting VM to snapshot", snapshot, "...")
err = prl.RevertToSnapshot(s.vmName, snapshot)
if err != nil {
return err
}
return nil
}
func (s *executor) createVM() error {
baseImage := s.Config.Parallels.BaseName
if baseImage == "" {
return errors.New("missing Image setting from Parallels config")
}
templateName := s.Config.Parallels.TemplateName
if templateName == "" {
templateName = baseImage + "-template"
}
// remove invalid template (removed?)
templateStatus, _ := prl.Status(templateName)
if templateStatus == prl.Invalid {
_ = prl.Unregister(templateName)
}
if !prl.Exist(templateName) {
s.Debugln("Creating template from VM", baseImage, "...")
err := prl.CreateTemplate(baseImage, templateName)
if err != nil {
return err
}
}
s.Debugln("Creating runner from VM template...")
err := prl.CreateOsVM(s.vmName, templateName)
if err != nil {
return err
}
s.Debugln("Bootstrapping VM...")
err = prl.Start(s.vmName)
if err != nil {
return err
}
// TODO: integration tests do fail on this due
// Unable to open new session in this virtual machine.
// Make sure the latest version of Parallels Tools is installed in this virtual machine and it has finished bootingg
s.Debugln("Waiting for VM to start...")
err = prl.TryExec(s.vmName, 120, "exit", "0")
if err != nil {
return err
}
s.Debugln("Waiting for VM to become responsive...")
err = s.verifyMachine(s.vmName)
if err != nil {
return err
}
return nil
}
func (s *executor) updateGuestTime() error {
s.Debugln("Updating VM date...")
timeServer := s.Config.Parallels.TimeServer
if timeServer == "" {
timeServer = "time.apple.com"
}
// Check either ntpdate command exists or not before trying to execute it
// Starting from Mojave ntpdate was removed
_, err := prl.Exec(s.vmName, "which", "ntpdate")
if err != nil {
// Fallback to sntp
return prl.TryExec(s.vmName, 20, "sudo", "sntp", "-sS", timeServer)
}
return prl.TryExec(s.vmName, 20, "sudo", "ntpdate", "-u", timeServer)
}
func (s *executor) Prepare(options common.ExecutorPrepareOptions) error {
err := s.AbstractExecutor.Prepare(options)
if err != nil {
return err
}
err = s.validateConfig()
if err != nil {
return err
}
err = s.printVersion()
if err != nil {
return err
}
unregisterInvalidVM(s.vmName)
s.vmName = s.getVMName()
if s.Config.Parallels.DisableSnapshots && prl.Exist(s.vmName) {
s.Debugln("Deleting old VM...")
killAndUnregisterVM(s.vmName)
}
s.tryRestoreFromSnapshot()
if !prl.Exist(s.vmName) {
s.Println("Creating new VM...")
err = s.createVM()
if err != nil {
return err
}
if !s.Config.Parallels.DisableSnapshots {
s.Println("Creating default snapshot...")
err = prl.CreateSnapshot(s.vmName, "Started")
if err != nil {
return err
}
}
}
err = s.ensureVMStarted()
if err != nil {
return err
}
return s.sshConnect()
}
func (s *executor) printVersion() error {
version, err := prl.Version()
if err != nil {
return err
}
s.Println("Using Parallels", version, "executor...")
return nil
}
func (s *executor) validateConfig() error {
if s.BuildShell.PassFile {
return errors.New("parallels doesn't support shells that require script file")
}
if s.Config.SSH == nil {
return errors.New("missing SSH configuration")
}
if s.Config.Parallels == nil {
return errors.New("missing Parallels configuration")
}
if s.Config.Parallels.BaseName == "" {
return errors.New("missing BaseName setting from Parallels config")
}
return nil
}
func (s *executor) tryRestoreFromSnapshot() {
if !prl.Exist(s.vmName) {
return
}
s.Println("Restoring VM from snapshot...")
err := s.restoreFromSnapshot()
if err != nil {
s.Println("Previous VM failed. Deleting, because", err)
killAndUnregisterVM(s.vmName)
}
}
func (s *executor) getVMName() string {
if s.Config.Parallels.DisableSnapshots {
return s.Config.Parallels.BaseName + "-" + s.Build.ProjectUniqueName()
}
return fmt.Sprintf(
"%s-runner-%s-concurrent-%d",
s.Config.Parallels.BaseName,
s.Build.Runner.ShortDescription(),
s.Build.RunnerID,
)
}
func unregisterInvalidVM(vmName string) {
// remove invalid VM (removed?)
vmStatus, _ := prl.Status(vmName)
if vmStatus == prl.Invalid {
_ = prl.Unregister(vmName)
}
}
func killAndUnregisterVM(vmName string) {
_ = prl.Kill(vmName)
_ = prl.Delete(vmName)
_ = prl.Unregister(vmName)
}
func (s *executor) ensureVMStarted() error {
s.Debugln("Checking VM status...")
status, err := prl.Status(s.vmName)
if err != nil {
return err
}
// Start VM if stopped
if status == prl.Stopped || status == prl.Suspended {
s.Println("Starting VM...")
err = prl.Start(s.vmName)
if err != nil {
return err
}
}
if status != prl.Running {
s.Debugln("Waiting for VM to run...")
err = prl.WaitForStatus(s.vmName, prl.Running, 60)
if err != nil {
return err
}
}
s.Println("Waiting for VM to become responsive...")
err = s.verifyMachine(s.vmName)
if err != nil {
return err
}
s.provisioned = true
// TODO: integration tests do fail on this due
// Unable to open new session in this virtual machine.
// Make sure the latest version of Parallels Tools is installed in this virtual machine and it has finished booting
err = s.updateGuestTime()
if err != nil {
s.Println("Could not sync with timeserver!")
return err
}
return nil
}
func (s *executor) sshConnect() error {
ipAddr, err := s.waitForIPAddress(s.vmName, 60)
if err != nil {
return err
}
s.Debugln("Starting SSH command...")
s.sshCommand = ssh.Client{
Config: *s.Config.SSH,
Stdout: s.Trace,
Stderr: s.Trace,
}
s.sshCommand.Host = ipAddr
s.Debugln("Connecting to SSH server...")
return s.sshCommand.Connect()
}
func (s *executor) Run(cmd common.ExecutorCommand) error {
err := s.sshCommand.Run(cmd.Context, ssh.Command{
Environment: s.BuildShell.Environment,
Command: s.BuildShell.GetCommandWithArguments(),
Stdin: cmd.Script,
})
if exitError, ok := err.(*ssh.ExitError); ok {
exitCode := exitError.ExitCode()
err = &common.BuildError{Inner: err, ExitCode: exitCode}
}
return err
}
func (s *executor) Cleanup() {
s.sshCommand.Cleanup()
if s.vmName != "" {
_ = prl.Kill(s.vmName)
if s.Config.Parallels.DisableSnapshots || !s.provisioned {
_ = prl.Delete(s.vmName)
}
}
s.AbstractExecutor.Cleanup()
}
func init() {
options := executors.ExecutorOptions{
DefaultCustomBuildsDirEnabled: false,
DefaultBuildsDir: "builds",
DefaultCacheDir: "cache",
SharedBuildsDir: false,
Shell: common.ShellScriptInfo{
Shell: "bash",
Type: common.LoginShell,
RunnerCommand: "gitlab-runner",
},
ShowHostname: true,
}
creator := func() common.Executor {
return &executor{
AbstractExecutor: executors.AbstractExecutor{
ExecutorOptions: options,
},
}
}
featuresUpdater := func(features *common.FeaturesInfo) {
features.Variables = true
}
common.RegisterExecutorProvider("parallels", executors.DefaultExecutorProvider{
Creator: creator,
FeaturesUpdater: featuresUpdater,
DefaultShellName: options.Shell.Shell,
})
}
package shell
import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"github.com/kardianos/osext"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors"
"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
"gitlab.com/gitlab-org/gitlab-runner/helpers/process"
)
var newProcessKillWaiter = process.NewOSKillWait
var newCommander = process.NewOSCmd
type executor struct {
executors.AbstractExecutor
}
func (s *executor) Prepare(options common.ExecutorPrepareOptions) error {
if options.User != "" {
s.Shell().User = options.User
}
// expand environment variables to have current directory
wd, err := os.Getwd()
if err != nil {
return fmt.Errorf("getwd: %w", err)
}
mapping := func(key string) string {
switch key {
case "PWD":
return wd
default:
return ""
}
}
s.DefaultBuildsDir = os.Expand(s.DefaultBuildsDir, mapping)
s.DefaultCacheDir = os.Expand(s.DefaultCacheDir, mapping)
// Pass control to executor
err = s.AbstractExecutor.Prepare(options)
if err != nil {
return err
}
s.Println("Using Shell executor...")
return nil
}
func (s *executor) Run(cmd common.ExecutorCommand) error {
s.BuildLogger.Debugln("Using new shell command execution")
cmdOpts := process.CommandOptions{
Env: append(os.Environ(), s.BuildShell.Environment...),
Stdout: s.Trace,
Stderr: s.Trace,
UseWindowsLegacyProcessStrategy: s.Build.IsFeatureFlagOn(featureflags.UseWindowsLegacyProcessStrategy),
}
args := s.BuildShell.Arguments
stdin, args, cleanup, err := s.shellScriptArgs(cmd, args)
if err != nil {
return err
}
defer cleanup()
cmdOpts.Stdin = stdin
// Create execution command
c := newCommander(s.BuildShell.Command, args, cmdOpts)
// Start a process
err = c.Start()
if err != nil {
return fmt.Errorf("failed to start process: %w", err)
}
// Wait for process to finish
waitCh := make(chan error, 1)
go func() {
waitErr := c.Wait()
var exitErr *exec.ExitError
if errors.As(waitErr, &exitErr) {
waitErr = &common.BuildError{Inner: waitErr, ExitCode: exitErr.ExitCode()}
}
waitCh <- waitErr
}()
// Support process abort
select {
case err = <-waitCh:
return err
case <-cmd.Context.Done():
logger := common.NewProcessLoggerAdapter(s.BuildLogger)
return newProcessKillWaiter(logger, s.Config.GetGracefulKillTimeout(), s.Config.GetForceKillTimeout()).
KillAndWait(c, waitCh)
}
}
func (s *executor) shellScriptArgs(cmd common.ExecutorCommand, args []string) (io.Reader, []string, func(), error) {
if !s.BuildShell.PassFile {
return strings.NewReader(cmd.Script), args, func() {}, nil
}
scriptDir, err := ioutil.TempDir("", "build_script")
if err != nil {
return nil, nil, func() {}, fmt.Errorf("creating tmp build script dir: %w", err)
}
cleanup := func() {
err := os.RemoveAll(scriptDir)
if err != nil {
s.BuildLogger.Warningln("Failed to remove build script directory", scriptDir, err)
}
}
scriptFile := filepath.Join(scriptDir, "script."+s.BuildShell.Extension)
err = ioutil.WriteFile(scriptFile, []byte(cmd.Script), 0700)
if err != nil {
return nil, nil, cleanup, fmt.Errorf("writing script file: %w", err)
}
return nil, append(args, scriptFile), cleanup, nil
}
func init() {
// Look for self
runnerCommand, err := osext.Executable()
if err != nil {
logrus.Warningln(err)
}
RegisterExecutor("shell", runnerCommand)
}
func RegisterExecutor(executorName string, runnerCommandPath string) {
options := executors.ExecutorOptions{
DefaultCustomBuildsDirEnabled: false,
DefaultBuildsDir: "$PWD/builds",
DefaultCacheDir: "$PWD/cache",
SharedBuildsDir: true,
Shell: common.ShellScriptInfo{
Shell: common.GetDefaultShell(),
Type: common.LoginShell,
RunnerCommand: runnerCommandPath,
},
ShowHostname: false,
}
creator := func() common.Executor {
return &executor{
AbstractExecutor: executors.AbstractExecutor{
ExecutorOptions: options,
},
}
}
featuresUpdater := func(features *common.FeaturesInfo) {
features.Variables = true
features.Shared = true
if runtime.GOOS != "windows" {
features.Session = true
features.Terminal = true
}
}
common.RegisterExecutorProvider(executorName, executors.DefaultExecutorProvider{
Creator: creator,
FeaturesUpdater: featuresUpdater,
DefaultShellName: options.Shell.Shell,
})
}
// +build !windows
package shell
import (
"errors"
"net/http"
"os"
"os/exec"
"github.com/kr/pty"
terminalsession "gitlab.com/gitlab-org/gitlab-runner/session/terminal"
terminal "gitlab.com/gitlab-org/gitlab-terminal"
)
type terminalConn struct {
shellFd *os.File
}
func (t terminalConn) Start(w http.ResponseWriter, r *http.Request, timeoutCh, disconnectCh chan error) {
proxy := terminal.NewFileDescriptorProxy(1) // one stopper: terminal exit handler
terminalsession.ProxyTerminal(
timeoutCh,
disconnectCh,
proxy.StopCh,
func() {
terminal.ProxyFileDescriptor(w, r, t.shellFd, proxy)
},
)
}
func (t terminalConn) Close() error {
return t.shellFd.Close()
}
func (s *executor) Connect() (terminalsession.Conn, error) {
if s.Shell().Shell == "pwsh" {
return nil, errors.New("not yet supported")
}
cmd := exec.Command(s.BuildShell.Command, s.BuildShell.Arguments...)
if cmd == nil {
return nil, errors.New("failed to generate shell command")
}
shellFD, err := pty.Start(cmd)
if err != nil {
return nil, err
}
session := terminalConn{shellFd: shellFD}
return session, nil
}
package ssh
import (
"errors"
"fmt"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors"
"gitlab.com/gitlab-org/gitlab-runner/helpers/ssh"
)
type executor struct {
executors.AbstractExecutor
sshCommand ssh.Client
}
func (s *executor) Prepare(options common.ExecutorPrepareOptions) error {
err := s.AbstractExecutor.Prepare(options)
if err != nil {
return fmt.Errorf("prearing AbstractExecutor: %w", err)
}
s.Println("Using SSH executor...")
if s.BuildShell.PassFile {
return errors.New("SSH doesn't support shells that require script file")
}
if s.Config.SSH == nil {
return errors.New("missing SSH configuration")
}
s.Debugln("Starting SSH command...")
// Create SSH command
s.sshCommand = ssh.Client{
Config: *s.Config.SSH,
Stdout: s.Trace,
Stderr: s.Trace,
}
s.Debugln("Connecting to SSH server...")
err = s.sshCommand.Connect()
if err != nil {
return fmt.Errorf("ssh command Connect() error: %w", err)
}
return nil
}
func (s *executor) Run(cmd common.ExecutorCommand) error {
err := s.sshCommand.Run(cmd.Context, ssh.Command{
Environment: s.BuildShell.Environment,
Command: s.BuildShell.GetCommandWithArguments(),
Stdin: cmd.Script,
})
if exitError, ok := err.(*ssh.ExitError); ok {
exitCode := exitError.ExitCode()
err = &common.BuildError{Inner: err, ExitCode: exitCode}
}
return err
}
func (s *executor) Cleanup() {
s.sshCommand.Cleanup()
s.AbstractExecutor.Cleanup()
}
func init() {
options := executors.ExecutorOptions{
DefaultCustomBuildsDirEnabled: false,
DefaultBuildsDir: "builds",
DefaultCacheDir: "cache",
SharedBuildsDir: true,
Shell: common.ShellScriptInfo{
Shell: "bash",
Type: common.LoginShell,
RunnerCommand: "gitlab-runner",
},
ShowHostname: true,
}
creator := func() common.Executor {
return &executor{
AbstractExecutor: executors.AbstractExecutor{
ExecutorOptions: options,
},
}
}
featuresUpdater := func(features *common.FeaturesInfo) {
features.Variables = true
features.Shared = true
}
common.RegisterExecutorProvider("ssh", executors.DefaultExecutorProvider{
Creator: creator,
FeaturesUpdater: featuresUpdater,
DefaultShellName: options.Shell.Shell,
})
}
package ssh
import (
"fmt"
"net"
"strconv"
"strings"
"github.com/tevino/abool"
cryptoSSH "golang.org/x/crypto/ssh"
)
type StubSSHServer struct {
User string
Password string
Config *cryptoSSH.ServerConfig
stop chan bool
shouldExit *abool.AtomicBool
}
func NewStubServer(user, pass string, privateKey []byte) (*StubSSHServer, error) {
server := &StubSSHServer{
User: user,
Password: pass,
Config: &cryptoSSH.ServerConfig{
PasswordCallback: func(conn cryptoSSH.ConnMetadata, password []byte) (*cryptoSSH.Permissions, error) {
if conn.User() == user && string(password) == pass {
return nil, nil
}
return nil, fmt.Errorf("wrong password for %q", conn.User())
},
},
stop: make(chan bool),
shouldExit: abool.New(),
}
key, err := cryptoSSH.ParsePrivateKey(privateKey)
if err != nil {
return nil, err
}
server.Config.AddHostKey(key)
return server, nil
}
func (s *StubSSHServer) Start() (int, error) {
listener, err := net.Listen("tcp", "127.0.0.1:")
if err != nil {
return 0, err
}
go func() {
<-s.stop
s.shouldExit.Set()
_ = listener.Close()
}()
address := strings.SplitN(listener.Addr().String(), ":", 2)
go s.mainLoop(listener)
return strconv.Atoi(address[1])
}
func (s *StubSSHServer) Stop() {
s.stop <- true
}
func (s *StubSSHServer) mainLoop(listener net.Listener) {
for {
if s.shouldExit.IsSet() {
return
}
conn, err := listener.Accept()
if err != nil {
continue
}
if s.shouldExit.IsSet() {
return
}
// upgrade to ssh connection
_, _, _, _ = cryptoSSH.NewServerConn(conn, s.Config)
// This is enough just for handling incoming connections
}
}
package virtualbox
import (
"errors"
"fmt"
"time"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/executors"
"gitlab.com/gitlab-org/gitlab-runner/helpers/ssh"
vbox "gitlab.com/gitlab-org/gitlab-runner/helpers/virtualbox"
)
type executor struct {
executors.AbstractExecutor
vmName string
sshCommand ssh.Client
sshPort string
provisioned bool
machineVerified bool
}
func (s *executor) verifyMachine(sshPort string) error {
if s.machineVerified {
return nil
}
// Create SSH command
sshCommand := ssh.Client{
Config: *s.Config.SSH,
Stdout: s.Trace,
Stderr: s.Trace,
ConnectRetries: 30,
}
sshCommand.Port = sshPort
sshCommand.Host = "localhost"
s.Debugln("Connecting to SSH...")
err := sshCommand.Connect()
if err != nil {
return err
}
defer sshCommand.Cleanup()
err = sshCommand.Run(s.Context, ssh.Command{Command: []string{"exit"}})
if err != nil {
return err
}
s.machineVerified = true
return nil
}
func (s *executor) restoreFromSnapshot() error {
s.Debugln("Reverting VM to current snapshot...")
err := vbox.RevertToSnapshot(s.vmName)
if err != nil {
return err
}
return nil
}
func (s *executor) determineBaseSnapshot(baseImage string) string {
var err error
baseSnapshot := s.Config.VirtualBox.BaseSnapshot
if baseSnapshot == "" {
baseSnapshot, err = vbox.GetCurrentSnapshot(baseImage)
if err != nil {
if s.Config.VirtualBox.DisableSnapshots {
s.Debugln("No snapshots found for base VM", baseImage)
return ""
}
baseSnapshot = "Base State"
}
}
if baseSnapshot != "" && !vbox.HasSnapshot(baseImage, baseSnapshot) {
if s.Config.VirtualBox.DisableSnapshots {
s.Warningln("Snapshot", baseSnapshot, "not found in base VM", baseImage)
return ""
}
s.Debugln("Creating snapshot", baseSnapshot, "from current base VM", baseImage, "state...")
err = vbox.CreateSnapshot(baseImage, baseSnapshot)
if err != nil {
s.Warningln("Failed to create snapshot", baseSnapshot, "from base VM", baseImage)
return ""
}
}
return baseSnapshot
}
// virtualbox doesn't support templates
func (s *executor) createVM(vmName string) (err error) {
baseImage := s.Config.VirtualBox.BaseName
if baseImage == "" {
return errors.New("missing Image setting from VirtualBox configuration")
}
_, err = vbox.Status(vmName)
if err != nil {
_ = vbox.Unregister(vmName)
}
if !vbox.Exist(vmName) {
baseSnapshot := s.determineBaseSnapshot(baseImage)
if baseSnapshot == "" {
s.Debugln("Creating testing VM from VM", baseImage, "...")
} else {
s.Debugln("Creating testing VM from VM", baseImage, "snapshot", baseSnapshot, "...")
}
err = vbox.CreateOsVM(baseImage, vmName, baseSnapshot, s.Config.VirtualBox.BaseFolder)
if err != nil {
return err
}
}
s.Debugln("Identify SSH Port...")
s.sshPort, err = vbox.FindSSHPort(s.vmName)
if err != nil {
s.Debugln("Creating localhost ssh forwarding...")
vmSSHPort := s.Config.SSH.Port
if vmSSHPort == "" {
vmSSHPort = "22"
}
s.sshPort, err = vbox.ConfigureSSH(vmName, vmSSHPort)
if err != nil {
return err
}
}
s.Debugln("Using local", s.sshPort, "SSH port to connect to VM...")
s.Debugln("Bootstraping VM...")
err = vbox.Start(s.vmName)
if err != nil {
return err
}
s.Debugln("Waiting for VM to become responsive...")
time.Sleep(10 * time.Second)
err = s.verifyMachine(s.sshPort)
if err != nil {
return err
}
return nil
}
func (s *executor) Prepare(options common.ExecutorPrepareOptions) error {
err := s.AbstractExecutor.Prepare(options)
if err != nil {
return err
}
err = s.validateConfig()
if err != nil {
return err
}
err = s.printVersion()
if err != nil {
return err
}
s.vmName = s.getVMName()
if s.Config.VirtualBox.DisableSnapshots && vbox.Exist(s.vmName) {
s.Debugln("Deleting old VM...")
killAndUnregisterVM(s.vmName)
}
s.tryRestoreFromSnapshot()
if !vbox.Exist(s.vmName) {
s.Println("Creating new VM...")
err = s.createVM(s.vmName)
if err != nil {
return err
}
if !s.Config.VirtualBox.DisableSnapshots {
s.Println("Creating default snapshot...")
err = vbox.CreateSnapshot(s.vmName, "Started")
if err != nil {
return err
}
}
}
err = s.ensureVMStarted()
if err != nil {
return err
}
return s.sshConnect()
}
func (s *executor) printVersion() error {
version, err := vbox.Version()
if err != nil {
return err
}
s.Println("Using VirtualBox version", version, "executor...")
return nil
}
func (s *executor) validateConfig() error {
if s.BuildShell.PassFile {
return errors.New("virtualbox doesn't support shells that require script file")
}
if s.Config.SSH == nil {
return errors.New("missing SSH config")
}
if s.Config.VirtualBox == nil {
return errors.New("missing VirtualBox configuration")
}
if s.Config.VirtualBox.BaseName == "" {
return errors.New("missing BaseName setting from VirtualBox configuration")
}
return nil
}
func (s *executor) getVMName() string {
if s.Config.VirtualBox.DisableSnapshots {
return s.Config.VirtualBox.BaseName + "-" + s.Build.ProjectUniqueName()
}
return fmt.Sprintf(
"%s-runner-%s-concurrent-%d",
s.Config.VirtualBox.BaseName,
s.Build.Runner.ShortDescription(),
s.Build.RunnerID,
)
}
func (s *executor) tryRestoreFromSnapshot() {
if !vbox.Exist(s.vmName) {
return
}
s.Println("Restoring VM from snapshot...")
err := s.restoreFromSnapshot()
if err != nil {
s.Println("Previous VM failed. Deleting, because", err)
killAndUnregisterVM(s.vmName)
}
}
func killAndUnregisterVM(vmName string) {
_ = vbox.Kill(vmName)
_ = vbox.Delete(vmName)
_ = vbox.Unregister(vmName)
}
func (s *executor) ensureVMStarted() error {
s.Debugln("Checking VM status...")
status, err := vbox.Status(s.vmName)
if err != nil {
return err
}
if !vbox.IsStatusOnlineOrTransient(status) {
s.Println("Starting VM...")
err = vbox.Start(s.vmName)
if err != nil {
return err
}
}
if status != vbox.Running {
s.Debugln("Waiting for VM to run...")
err = vbox.WaitForStatus(s.vmName, vbox.Running, 60)
if err != nil {
return err
}
}
s.Debugln("Identify SSH Port...")
sshPort, err := vbox.FindSSHPort(s.vmName)
s.sshPort = sshPort
if err != nil {
return err
}
s.Println("Waiting for VM to become responsive...")
err = s.verifyMachine(s.sshPort)
if err != nil {
return err
}
s.provisioned = true
return nil
}
func (s *executor) sshConnect() error {
s.Println("Starting SSH command...")
s.sshCommand = ssh.Client{
Config: *s.Config.SSH,
Stdout: s.Trace,
Stderr: s.Trace,
}
s.sshCommand.Port = s.sshPort
s.sshCommand.Host = "localhost"
s.Debugln("Connecting to SSH server...")
return s.sshCommand.Connect()
}
func (s *executor) Run(cmd common.ExecutorCommand) error {
err := s.sshCommand.Run(cmd.Context, ssh.Command{
Environment: s.BuildShell.Environment,
Command: s.BuildShell.GetCommandWithArguments(),
Stdin: cmd.Script,
})
if exitError, ok := err.(*ssh.ExitError); ok {
exitCode := exitError.ExitCode()
err = &common.BuildError{Inner: err, ExitCode: exitCode}
}
return err
}
func (s *executor) Cleanup() {
s.sshCommand.Cleanup()
if s.vmName != "" {
_ = vbox.Kill(s.vmName)
if s.Config.VirtualBox.DisableSnapshots || !s.provisioned {
_ = vbox.Delete(s.vmName)
}
}
}
func init() {
options := executors.ExecutorOptions{
DefaultCustomBuildsDirEnabled: false,
DefaultBuildsDir: "builds",
DefaultCacheDir: "cache",
SharedBuildsDir: false,
Shell: common.ShellScriptInfo{
Shell: "bash",
Type: common.LoginShell,
RunnerCommand: "gitlab-runner",
},
ShowHostname: true,
}
creator := func() common.Executor {
return &executor{
AbstractExecutor: executors.AbstractExecutor{
ExecutorOptions: options,
},
}
}
featuresUpdater := func(features *common.FeaturesInfo) {
features.Variables = true
}
common.RegisterExecutorProvider("virtualbox", executors.DefaultExecutorProvider{
Creator: creator,
FeaturesUpdater: featuresUpdater,
DefaultShellName: options.Shell.Shell,
})
}
package archives
import (
"fmt"
"io"
"os"
gzip "github.com/klauspost/pgzip"
"github.com/sirupsen/logrus"
)
func writeGzipFile(w io.Writer, fileName string, fileInfo os.FileInfo) error {
if !fileInfo.Mode().IsRegular() {
return fmt.Errorf("the %q is not a regular file", fileName)
}
gz := gzip.NewWriter(w)
gz.Header.Name = fileInfo.Name()
gz.Header.Comment = fileName
gz.Header.ModTime = fileInfo.ModTime()
defer func() { _ = gz.Close() }()
file, err := os.Open(fileName)
if err != nil {
return err
}
defer func() { _ = file.Close() }()
_, err = io.Copy(gz, file)
return err
}
func CreateGzipArchive(w io.Writer, fileNames []string) error {
for _, fileName := range fileNames {
fi, err := os.Lstat(fileName)
if os.IsNotExist(err) {
logrus.Warningln("File ignored:", err)
continue
} else if err != nil {
return err
}
err = writeGzipFile(w, fileName, fi)
if err != nil {
return err
}
}
return nil
}
package archives
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/sirupsen/logrus"
)
func isPathAGitDirectory(path string) bool {
parts := strings.Split(filepath.Clean(path), string(filepath.Separator))
if len(parts) > 0 && parts[0] == ".git" {
return true
}
return false
}
func errorIfGitDirectory(path string) *os.PathError {
if !isPathAGitDirectory(path) {
return nil
}
return &os.PathError{
Op: ".git inside of archive",
Path: path,
Err: errors.New("trying to archive or extract .git path"),
}
}
func printGitArchiveWarning(operation string) {
logrus.Warn(fmt.Sprintf("Part of .git directory is on the list of files to %s", operation))
logrus.Warn("This may introduce unexpected problems")
}
package archives
import (
"os"
"sync"
)
// When extracting an archive, the same PathError.Op may be repeated for every
// file in the archive; use pathErrorTracker to suppress repetitious log output
type pathErrorTracker struct {
lock sync.Mutex
seenOps map[string]bool
}
// check whether the error is actionable, which is to say, not nil and either
// not a PathError, or a novel PathError
func (p *pathErrorTracker) actionable(e error) bool {
pathErr, isPathErr := e.(*os.PathError)
if e == nil || isPathErr && pathErr == nil {
return false
}
if !isPathErr {
return true
}
p.lock.Lock()
defer p.lock.Unlock()
seen := p.seenOps[pathErr.Op]
p.seenOps[pathErr.Op] = true
// actionable if *not* seen before
return !seen
}
func newPathErrorTracker() *pathErrorTracker {
return &pathErrorTracker{
seenOps: make(map[string]bool),
}
}
package archives
import (
"archive/zip"
"io"
"os"
"github.com/sirupsen/logrus"
)
func createZipDirectoryEntry(archive *zip.Writer, fh *zip.FileHeader) error {
fh.Name += "/"
_, err := archive.CreateHeader(fh)
return err
}
func createZipSymlinkEntry(archive *zip.Writer, fh *zip.FileHeader) error {
fw, err := archive.CreateHeader(fh)
if err != nil {
return err
}
link, err := os.Readlink(fh.Name)
if err != nil {
return err
}
_, err = io.WriteString(fw, link)
return err
}
func createZipFileEntry(archive *zip.Writer, fh *zip.FileHeader) error {
fh.Method = zip.Deflate
fw, err := archive.CreateHeader(fh)
if err != nil {
return err
}
file, err := os.Open(fh.Name)
if err != nil {
return err
}
_, err = io.Copy(fw, file)
_ = file.Close()
if err != nil {
return err
}
return nil
}
func createZipEntry(archive *zip.Writer, fileName string) error {
fi, err := os.Lstat(fileName)
if err != nil {
logrus.Warningln("File ignored:", err)
return nil
}
fh, err := zip.FileInfoHeader(fi)
if err != nil {
return err
}
fh.Name = fileName
fh.Extra = createZipExtra(fi)
// Set EFS flag to indicate that filenames and comments are UTF-8 encoded
fh.Flags |= 0x800
switch fi.Mode() & os.ModeType {
case os.ModeDir:
return createZipDirectoryEntry(archive, fh)
case os.ModeSymlink:
return createZipSymlinkEntry(archive, fh)
case os.ModeNamedPipe, os.ModeSocket, os.ModeDevice:
// Ignore the files that of these types
logrus.Warningln("File ignored:", fileName)
return nil
default:
return createZipFileEntry(archive, fh)
}
}
func CreateZipArchive(w io.Writer, fileNames []string) error {
tracker := newPathErrorTracker()
archive := zip.NewWriter(w)
defer func() { _ = archive.Close() }()
for _, fileName := range fileNames {
if err := errorIfGitDirectory(fileName); tracker.actionable(err) {
printGitArchiveWarning("archive")
}
err := createZipEntry(archive, fileName)
if err != nil {
return err
}
}
return nil
}
package archives
import (
"archive/zip"
"bytes"
"encoding/binary"
"io"
"os"
"time"
)
const ZipUIDGidFieldType = 0x7875
const ZipTimestampFieldType = 0x5455
// ZipExtraField is taken from https://github.com/LuaDist/zip/blob/3.0/proginfo/extrafld.txt
type ZipExtraField struct {
Type uint16
Size uint16
}
type ZipUIDGidField struct {
Version uint8
UIDSize uint8
UID uint32
GIDSize uint8
Gid uint32
}
type ZipTimestampField struct {
Flags uint8
ModTime uint32
}
func createZipTimestampField(w io.Writer, fi os.FileInfo) (err error) {
tsField := ZipTimestampField{
1,
uint32(fi.ModTime().Unix()),
}
tsFieldType := ZipExtraField{
Type: ZipTimestampFieldType,
Size: uint16(binary.Size(&tsField)),
}
err = binary.Write(w, binary.LittleEndian, &tsFieldType)
if err == nil {
err = binary.Write(w, binary.LittleEndian, &tsField)
}
return
}
func processZipTimestampField(data []byte, file *zip.FileHeader) error {
if !file.Mode().IsDir() && !file.Mode().IsRegular() {
return nil
}
var tsField ZipTimestampField
err := binary.Read(bytes.NewReader(data), binary.LittleEndian, &tsField)
if err != nil {
return err
}
if (tsField.Flags & 1) == 1 {
modTime := time.Unix(int64(tsField.ModTime), 0)
acTime := time.Now()
return os.Chtimes(file.Name, acTime, modTime)
}
return nil
}
func createZipExtra(fi os.FileInfo) []byte {
var buffer bytes.Buffer
err := createZipUIDGidField(&buffer, fi)
if err == nil {
err = createZipTimestampField(&buffer, fi)
}
if err == nil {
return buffer.Bytes()
}
return nil
}
func readZipExtraField(r io.Reader) (field ZipExtraField, data []byte, err error) {
err = binary.Read(r, binary.LittleEndian, &field)
if err != nil {
return
}
data = make([]byte, field.Size)
_, err = r.Read(data)
if err != nil {
return
}
return
}
func processZipExtra(file *zip.FileHeader) error {
if len(file.Extra) == 0 {
return nil
}
r := bytes.NewReader(file.Extra)
for {
field, data, err := readZipExtraField(r)
if err == io.EOF {
break
} else if err != nil {
return err
}
switch field.Type {
case ZipUIDGidFieldType:
err = processZipUIDGidField(data, file)
case ZipTimestampFieldType:
err = processZipTimestampField(data, file)
}
if err != nil {
return err
}
}
return nil
}
// +build linux darwin freebsd openbsd
package archives
import (
"archive/zip"
"bytes"
"encoding/binary"
"errors"
"io"
"os"
"syscall"
)
func createZipUIDGidField(w io.Writer, fi os.FileInfo) (err error) {
stat, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
return
}
ugField := ZipUIDGidField{
1,
4, stat.Uid,
4, stat.Gid,
}
ugFieldType := ZipExtraField{
Type: ZipUIDGidFieldType,
Size: uint16(binary.Size(&ugField)),
}
err = binary.Write(w, binary.LittleEndian, &ugFieldType)
if err == nil {
err = binary.Write(w, binary.LittleEndian, &ugField)
}
return err
}
func processZipUIDGidField(data []byte, file *zip.FileHeader) error {
var ugField ZipUIDGidField
err := binary.Read(bytes.NewReader(data), binary.LittleEndian, &ugField)
if err != nil {
return err
}
if !(ugField.Version == 1 && ugField.UIDSize == 4 && ugField.GIDSize == 4) {
return errors.New("uid/gid data not supported")
}
return os.Lchown(file.Name, int(ugField.UID), int(ugField.Gid))
}
package archives
import (
"archive/zip"
"io"
"io/ioutil"
"os"
"path/filepath"
"github.com/sirupsen/logrus"
)
func extractZipDirectoryEntry(file *zip.File) (err error) {
err = os.Mkdir(file.Name, file.Mode().Perm())
// The error that directory does exists is not a error for us
if os.IsExist(err) {
err = nil
}
return
}
func extractZipSymlinkEntry(file *zip.File) (err error) {
var data []byte
in, err := file.Open()
if err != nil {
return err
}
defer func() { _ = in.Close() }()
data, err = ioutil.ReadAll(in)
if err != nil {
return err
}
// Remove symlink before creating a new one, otherwise we can error that file does exist
_ = os.Remove(file.Name)
err = os.Symlink(string(data), file.Name)
return
}
func extractZipFileEntry(file *zip.File) (err error) {
var out *os.File
in, err := file.Open()
if err != nil {
return err
}
defer func() { _ = in.Close() }()
// Remove file before creating a new one, otherwise we can error that file does exist
_ = os.Remove(file.Name)
out, err = os.OpenFile(file.Name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode().Perm())
if err != nil {
return err
}
defer func() { _ = out.Close() }()
_, err = io.Copy(out, in)
return
}
func extractZipFile(file *zip.File) (err error) {
// Create all parents to extract the file
err = os.MkdirAll(filepath.Dir(file.Name), 0777)
if err != nil {
return err
}
switch file.Mode() & os.ModeType {
case os.ModeDir:
err = extractZipDirectoryEntry(file)
case os.ModeSymlink:
err = extractZipSymlinkEntry(file)
case os.ModeNamedPipe, os.ModeSocket, os.ModeDevice:
// Ignore the files that of these types
logrus.Warningf("File ignored: %q", file.Name)
default:
err = extractZipFileEntry(file)
}
return
}
func ExtractZipArchive(archive *zip.Reader) error {
tracker := newPathErrorTracker()
for _, file := range archive.File {
if err := errorIfGitDirectory(file.Name); tracker.actionable(err) {
printGitArchiveWarning("extract")
}
if err := extractZipFile(file); tracker.actionable(err) {
logrus.Warningf("%s: %s (suppressing repeats)", file.Name, err)
}
}
for _, file := range archive.File {
// Update file permissions
if err := os.Chmod(file.Name, file.Mode().Perm()); tracker.actionable(err) {
logrus.Warningf("%s: %s (suppressing repeats)", file.Name, err)
}
// Process zip metadata
if err := processZipExtra(&file.FileHeader); tracker.actionable(err) {
logrus.Warningf("%s: %s (suppressing repeats)", file.Name, err)
}
}
return nil
}
func ExtractZipFile(fileName string) error {
archive, err := zip.OpenReader(fileName)
if err != nil {
return err
}
defer func() { _ = archive.Close() }()
return ExtractZipArchive(&archive.Reader)
}
package helpers
import (
"fmt"
"time"
)
type RawLogger interface {
SendRawLog(args ...interface{})
}
type BuildSection struct {
Name string
SkipMetrics bool
Run func() error
}
const (
traceSectionStart = "section_start:%v:%s\r" + ANSI_CLEAR
traceSectionEnd = "section_end:%v:%s\r" + ANSI_CLEAR
)
func nowUnixUTC() int64 {
return time.Now().UTC().Unix()
}
func (s *BuildSection) timestamp(format string, logger RawLogger) {
if s.SkipMetrics {
return
}
sectionLine := fmt.Sprintf(format, nowUnixUTC(), s.Name)
logger.SendRawLog(sectionLine)
}
func (s *BuildSection) start(logger RawLogger) {
s.timestamp(traceSectionStart, logger)
}
func (s *BuildSection) end(logger RawLogger) {
s.timestamp(traceSectionEnd, logger)
}
func (s *BuildSection) Execute(logger RawLogger) error {
s.start(logger)
defer s.end(logger)
return s.Run()
}
package certificate
import (
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"math/big"
"net"
"time"
)
const (
x509CertificatePrivateKeyBits = 2048
x509CertificateExpiryInYears = 2
x509CertificateSerialNumber = 1
x509CertificateOrganization = "GitLab Runner"
)
type X509Generator struct{}
func (c X509Generator) Generate(host string) (tls.Certificate, []byte, error) {
priv, err := rsa.GenerateKey(rand.Reader, x509CertificatePrivateKeyBits)
if err != nil {
return tls.Certificate{}, []byte{}, err
}
template := x509.Certificate{
SerialNumber: big.NewInt(x509CertificateSerialNumber),
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(x509CertificateExpiryInYears, 0, 0),
Subject: pkix.Name{
Organization: []string{x509CertificateOrganization},
},
KeyUsage: x509.KeyUsageKeyEncipherment |
x509.KeyUsageDigitalSignature |
x509.KeyUsageDataEncipherment,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
}
if ip := net.ParseIP(host); ip != nil {
template.IPAddresses = append(template.IPAddresses, ip)
} else {
template.DNSNames = append(template.DNSNames, host)
}
publicKeyBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, priv.Public(), priv)
if err != nil {
return tls.Certificate{}, []byte{}, errors.New("failed to create certificate")
}
publicKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: publicKeyBytes})
privateKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
parsedCertificate, err := tls.X509KeyPair(publicKeyPEM, privateKeyPEM)
if err != nil {
return tls.Certificate{}, []byte{}, err
}
return parsedCertificate, publicKeyPEM, nil
}
package cli_helpers
import (
"os"
"runtime/pprof"
"github.com/urfave/cli"
)
func SetupCPUProfile(app *cli.App) {
app.Flags = append(app.Flags, cli.StringFlag{
Name: "cpuprofile",
Usage: "write cpu profile to file",
EnvVar: "CPU_PROFILE",
})
appBefore := app.Before
appAfter := app.After
app.Before = func(c *cli.Context) error {
if cpuProfile := c.String("cpuprofile"); cpuProfile != "" {
f, err := os.Create(cpuProfile)
if err != nil {
return err
}
_ = pprof.StartCPUProfile(f)
}
if appBefore != nil {
return appBefore(c)
}
return nil
}
app.After = func(c *cli.Context) error {
pprof.StopCPUProfile()
if appAfter != nil {
return appAfter(c)
}
return nil
}
}
package cli_helpers
import (
"fmt"
"os"
"github.com/docker/docker/pkg/homedir"
"github.com/urfave/cli"
)
func FixHOME(app *cli.App) {
appBefore := app.Before
app.Before = func(c *cli.Context) error {
// Fix home
if key := homedir.Key(); os.Getenv(key) == "" {
value := homedir.Get()
if value == "" {
return fmt.Errorf("the %q is not set", key)
}
_ = os.Setenv(key, value)
}
if appBefore != nil {
return appBefore(c)
}
return nil
}
}
// +build !windows
package cli_helpers
func InitCli() {}
package cli_helpers
import (
"os"
"runtime"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
func LogRuntimePlatform(app *cli.App) {
appBefore := app.Before
app.Before = func(c *cli.Context) error {
fields := logrus.Fields{
"os": runtime.GOOS,
"arch": runtime.GOARCH,
"version": common.VERSION,
"revision": common.REVISION,
"pid": os.Getpid(),
}
logrus.WithFields(fields).Info("Runtime platform")
if appBefore != nil {
return appBefore(c)
}
return nil
}
}
package cli_helpers
import (
"strings"
"github.com/sirupsen/logrus"
)
// WarnOnBool logs warning if args contains true or false
// github.com/urfave/cli breaks badly if boolean are set using --flag true instead of --flag=true or just --flag
// this is a simple check that warn the user about this if detects "true" or "false" alone in the arguments
func WarnOnBool(args []string) {
// we skip the first element because it contains the program name
for idx, a := range args[1:] {
arg := strings.ToLower(a)
if arg == "true" || arg == "false" {
supposedFlag := "--key"
if idx > 0 {
supposedFlag = args[idx]
}
logrus.Warningf("boolean parameters must be passed in the command line with %s=%s", supposedFlag, arg)
logrus.Warningln("parameters after this may be ignored")
break
}
}
}
package helperimage
import (
"fmt"
"gitlab.com/gitlab-org/gitlab-runner/helpers/docker/errors"
"gitlab.com/gitlab-org/gitlab-runner/shells"
)
const (
OSTypeLinux = "linux"
OSTypeWindows = "windows"
//nolint:lll
// DockerHubWarningMessage is the message that is printed to the user when
// it's using the helper image hosted in Docker Hub. It is up to the caller
// to print this message.
DockerHubWarningMessage = "Pulling GitLab Runner helper image from Docker Hub. " +
"Helper image is migrating to registry.gitlab.com, " +
"for more information see " +
"https://docs.gitlab.com/runner/configuration/advanced-configuration.html#migrate-helper-image-to-registrygitlabcom"
// DockerHubName is the name of the helper image hosted in Docker Hub.
DockerHubName = "gitlab/gitlab-runner-helper"
// GitLabRegistryName is the name of the helper image hosted in registry.gitlab.com.
GitLabRegistryName = "registry.gitlab.com/gitlab-org/gitlab-runner/gitlab-runner-helper"
// DefaultFlavor is the default flavor of image we use for the helper.
DefaultFlavor = "alpine"
headRevision = "HEAD"
latestImageRevision = "latest"
)
type Info struct {
OSType string
Architecture string
Name string
Tag string
IsSupportingLocalImport bool
Cmd []string
}
func (i Info) String() string {
return fmt.Sprintf("%s:%s", i.Name, i.Tag)
}
// Config specifies details about the consumer of this package that need to be
// taken in consideration when building Container.
type Config struct {
OSType string
Architecture string
OperatingSystem string
Shell string
GitLabRegistry bool
Flavor string
}
type creator interface {
Create(revision string, cfg Config) (Info, error)
}
var supportedOsTypesFactories = map[string]creator{
OSTypeWindows: new(windowsInfo),
OSTypeLinux: new(linuxInfo),
}
func Get(revision string, cfg Config) (Info, error) {
factory, ok := supportedOsTypesFactories[cfg.OSType]
if !ok {
return Info{}, errors.NewErrOSNotSupported(cfg.OSType)
}
info, err := factory.Create(imageRevision(revision), cfg)
info.OSType = cfg.OSType
return info, err
}
func imageRevision(revision string) string {
if revision != headRevision {
return revision
}
return latestImageRevision
}
func imageName(gitlabRegistry bool) string {
if gitlabRegistry {
return GitLabRegistryName
}
return DockerHubName
}
func getPowerShellCmd(shell string) []string {
if shell == "" {
shell = shells.SNPowershell
}
return shells.PowershellDockerCmd(shell)
}
package helperimage
import (
"fmt"
"runtime"
"gitlab.com/gitlab-org/gitlab-runner/shells"
)
const (
platformAmd64 = "amd64"
platformArm6vl = "armv6l"
platformArmv7l = "armv7l"
platformAarch64 = "aarch64"
archX8664 = "x86_64"
archArm = "arm"
archArm64 = "arm64"
)
var bashCmd = []string{"gitlab-runner-build"}
type linuxInfo struct{}
func (l *linuxInfo) Create(revision string, cfg Config) (Info, error) {
arch := l.architecture(cfg.Architecture)
if cfg.Flavor == "" {
cfg.Flavor = DefaultFlavor
}
// alpine is a special case: we don't add the flavor to the tag name
// for backwards compatibility purposes. It existed before flavors were
// introduced.
if cfg.Flavor == "alpine" {
cfg.Flavor = ""
}
prefix := ""
if cfg.Flavor != "" {
prefix = cfg.Flavor + "-"
}
shell := cfg.Shell
if shell == "" {
shell = "bash"
}
cmd := bashCmd
tag := fmt.Sprintf("%s%s-%s", prefix, arch, revision)
if shell == shells.SNPwsh {
cmd = getPowerShellCmd(shell)
tag = fmt.Sprintf("%s-%s", tag, shell)
}
return Info{
Architecture: arch,
Name: imageName(cfg.GitLabRegistry),
Tag: tag,
IsSupportingLocalImport: true,
Cmd: cmd,
}, nil
}
func (l *linuxInfo) architecture(arch string) string {
switch arch {
case platformArm6vl, platformArmv7l:
return archArm
case platformAarch64:
return archArm64
case platformAmd64:
return archX8664
}
if arch != "" {
return arch
}
switch runtime.GOARCH {
case platformAmd64:
return archX8664
default:
return runtime.GOARCH
}
}
package helperimage
import (
"fmt"
"gitlab.com/gitlab-org/gitlab-runner/helpers/container/windows"
)
const (
baseImage1809 = "servercore1809"
baseImage2004 = "servercore2004"
baseImage20H2 = "servercore20H2"
windowsSupportedArchitecture = "x86_64"
)
var helperImages = map[string]string{
windows.V1809: baseImage1809,
windows.V2004: baseImage2004,
windows.V20H2: baseImage20H2,
}
type windowsInfo struct{}
func (w *windowsInfo) Create(revision string, cfg Config) (Info, error) {
baseImage, err := w.baseImage(cfg.OperatingSystem)
if err != nil {
return Info{}, fmt.Errorf("detecting base image: %w", err)
}
return Info{
Architecture: windowsSupportedArchitecture,
Name: imageName(cfg.GitLabRegistry),
Tag: fmt.Sprintf("%s-%s-%s", windowsSupportedArchitecture, revision, baseImage),
IsSupportingLocalImport: false,
Cmd: getPowerShellCmd(cfg.Shell),
}, nil
}
func (w *windowsInfo) baseImage(operatingSystem string) (string, error) {
version, err := windows.Version(operatingSystem)
if err != nil {
return "", err
}
baseImage, ok := helperImages[version]
if !ok {
return "", windows.NewUnsupportedWindowsVersionError(operatingSystem)
}
return baseImage, nil
}
package services
import (
"fmt"
"regexp"
"strings"
"github.com/docker/distribution/reference"
)
type Service struct {
Service string
Version string
ImageName string
Aliases []string
}
var referenceRegexpNoPort = regexp.MustCompile(`^(.*?)(|:[0-9]+)(|/.*)$`)
const imageVersionLatest = "latest"
// SplitNameAndVersion parses Docker registry image urls and constructs a struct with correct
// image url, name, version and aliases
func SplitNameAndVersion(serviceDescription string) Service {
// Try to find matches in e.g. subdomain.domain.tld:8080/namespace/service:version
matches := reference.ReferenceRegexp.FindStringSubmatch(serviceDescription)
if len(matches) == 0 {
return Service{
ImageName: serviceDescription,
Version: imageVersionLatest,
}
}
// -> subdomain.domain.tld:8080/namespace/service
imageWithoutVersion := matches[1]
// -> version
imageVersion := matches[2]
registryMatches := referenceRegexpNoPort.FindStringSubmatch(imageWithoutVersion)
// -> subdomain.domain.tld
registry := registryMatches[1]
// -> /namespace/service
imageName := registryMatches[3]
service := Service{}
service.Service = registry + imageName
if len(imageVersion) > 0 {
service.ImageName = serviceDescription
service.Version = imageVersion
} else {
service.ImageName = fmt.Sprintf("%s:%s", imageWithoutVersion, imageVersionLatest)
service.Version = imageVersionLatest
}
alias := strings.ReplaceAll(service.Service, "/", "__")
service.Aliases = append(service.Aliases, alias)
// Create alternative link name according to RFC 1123
// Where you can use only `a-zA-Z0-9-`
alternativeName := strings.ReplaceAll(service.Service, "/", "-")
if alias != alternativeName {
service.Aliases = append(service.Aliases, alternativeName)
}
return service
}
package windows
import (
"fmt"
"strings"
)
const (
// V1809 is the Windows version that is 1809 and also known as Windows 2019
// ltsc.
V1809 = "1809"
// V2004 is the Windows version that is 2004 sac.
V2004 = "2004"
// V20H2 is the Windows version that is 2009 sac.
V20H2 = "2009"
)
// UnsupportedWindowsVersionError represents that the version specified is not
// supported.
type UnsupportedWindowsVersionError struct {
Version string
}
func NewUnsupportedWindowsVersionError(version string) *UnsupportedWindowsVersionError {
return &UnsupportedWindowsVersionError{Version: version}
}
func (e *UnsupportedWindowsVersionError) Error() string {
return fmt.Sprintf("unsupported Windows Version: %s", e.Version)
}
func (e *UnsupportedWindowsVersionError) Is(err error) bool {
_, ok := err.(*UnsupportedWindowsVersionError)
return ok
}
var supportedWindowsVersions = []string{
V1809,
V2004,
V20H2,
}
var supportedWindowsBuilds = map[string]string{
"10.0.17763": V1809,
"10.0.19041": V2004,
}
// Version checks the specified operatingSystem to see if it's one of the
// supported Windows version. If true, it returns the os version.
// UnsupportedWindowsVersionError is returned when no supported Windows version
// is found in the string.
func Version(operatingSystem string) (string, error) {
for _, windowsVersion := range supportedWindowsVersions {
if strings.Contains(operatingSystem, fmt.Sprintf(" %s ", windowsVersion)) {
return windowsVersion, nil
}
}
windowsVersion, ok := supportedWindowsBuilds[operatingSystem]
if ok {
return windowsVersion, nil
}
return "", NewUnsupportedWindowsVersionError(operatingSystem)
}
package helpers
import (
"bufio"
"bytes"
"github.com/BurntSushi/toml"
"gopkg.in/yaml.v2"
)
func ToYAML(src interface{}) string {
data, err := yaml.Marshal(src)
if err == nil {
return string(data)
}
return ""
}
func ToTOML(src interface{}) string {
var data bytes.Buffer
buffer := bufio.NewWriter(&data)
if err := toml.NewEncoder(buffer).Encode(src); err != nil {
return ""
}
if err := buffer.Flush(); err != nil {
return ""
}
return data.String()
}
func ToConfigMap(list interface{}) (map[string]interface{}, bool) {
x, ok := list.(map[string]interface{})
if ok {
return x, ok
}
y, ok := list.(map[interface{}]interface{})
if !ok {
return nil, false
}
result := make(map[string]interface{})
for k, v := range y {
result[k.(string)] = v
}
return result, true
}
func GetMapKey(value map[string]interface{}, keys ...string) (result interface{}, ok bool) {
result = value
for _, key := range keys {
switch t := result.(type) {
case map[string]interface{}:
if result, ok = t[key]; ok {
continue
}
case map[interface{}]interface{}:
if result, ok = t[key]; ok {
continue
}
}
return nil, false
}
return result, true
}
package test
import (
"regexp"
"testing"
"github.com/stretchr/testify/assert"
)
func AssertRFC1123Compatibility(t *testing.T, name string) {
dns1123MaxLength := 63
dns1123FormatRegexp := regexp.MustCompile("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$")
assert.True(t, len(name) <= dns1123MaxLength, "Name length needs to be shorter than %d", dns1123MaxLength)
assert.Regexp(t, dns1123FormatRegexp, name, "Name needs to be in RFC-1123 allowed format")
}
package dns
import (
"regexp"
"strings"
"k8s.io/apimachinery/pkg/util/validation"
)
const (
RFC1123NameMaximumLength = 63
RFC1123NotAllowedCharacters = "[^-a-z0-9]"
RFC1123NotAllowedStartCharacters = "^[^a-z0-9]+"
)
func MakeRFC1123Compatible(name string) string {
name = strings.ToLower(name)
nameNotAllowedChars := regexp.MustCompile(RFC1123NotAllowedCharacters)
name = nameNotAllowedChars.ReplaceAllString(name, "")
nameNotAllowedStartChars := regexp.MustCompile(RFC1123NotAllowedStartCharacters)
name = nameNotAllowedStartChars.ReplaceAllString(name, "")
if len(name) > RFC1123NameMaximumLength {
name = name[0:RFC1123NameMaximumLength]
}
return name
}
const emptyRFC1123SubdomainErrorMessage = "validating rfc1123 subdomain"
type RFC1123SubdomainError struct {
errs []string
}
func (d *RFC1123SubdomainError) Error() string {
if len(d.errs) == 0 {
return emptyRFC1123SubdomainErrorMessage
}
return strings.Join(d.errs, ", ")
}
func (d *RFC1123SubdomainError) Is(err error) bool {
_, ok := err.(*RFC1123SubdomainError)
return ok
}
func ValidateDNS1123Subdomain(name string) error {
errs := validation.IsDNS1123Subdomain(name)
if len(errs) == 0 {
return nil
}
return &RFC1123SubdomainError{errs: errs}
}
package auth
import (
"bytes"
"encoding/base64"
"encoding/json"
"errors"
"io"
"os"
"os/user"
"path/filepath"
"strings"
"github.com/docker/cli/cli/config/configfile"
"github.com/docker/cli/cli/config/credentials"
"github.com/docker/cli/cli/config/types"
"github.com/docker/docker/pkg/homedir"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
const (
// DefaultDockerRegistry is the name of the index
DefaultDockerRegistry = "docker.io"
authConfigSourceNameUserVariable = "$DOCKER_AUTH_CONFIG"
authConfigSourceNameJobPayload = "job payload (GitLab Registry)"
)
var (
HomeDirectory = homedir.Get()
errNoHomeDir = errors.New("no home directory found")
errPathTraversal = errors.New("path traversal is not allowed")
)
// RegistryInfo represents the source and authentication for a given registry.
type RegistryInfo struct {
Source string
AuthConfig types.AuthConfig
}
type authConfigResolver func() (string, map[string]types.AuthConfig, error)
// ResolveConfigForImage returns the auth configuration for a particular image.
// Returns nil on no config found.
// See ResolveConfigs for source information.
func ResolveConfigForImage(
imageName, dockerAuthConfig, username string,
credentials []common.Credentials,
) (*RegistryInfo, error) {
authConfigs, err := ResolveConfigs(dockerAuthConfig, username, credentials)
if len(authConfigs) == 0 || err != nil {
return nil, err
}
indexName, _ := splitDockerImageName(imageName)
info, ok := authConfigs[indexName]
if !ok {
return nil, nil
}
return &info, nil
}
// ResolveConfigs returns the authentication configuration for docker registries.
// Goes through several sources in this order:
// 1. DOCKER_AUTH_CONFIG
// 2. ~/.docker/config.json or .dockercfg
// 3. Build credentials
// Returns a map of registry hostname to RegistryInfo
func ResolveConfigs(
dockerAuthConfig, username string,
credentials []common.Credentials,
) (map[string]RegistryInfo, error) {
resolvers := []authConfigResolver{
func() (string, map[string]types.AuthConfig, error) {
return getUserConfiguration(dockerAuthConfig)
},
func() (string, map[string]types.AuthConfig, error) {
return getHomeDirConfiguration(username)
},
func() (string, map[string]types.AuthConfig, error) {
return getBuildConfiguration(credentials)
},
}
res := make(map[string]RegistryInfo)
for _, r := range resolvers {
source, configs, err := r()
if errors.Is(err, errPathTraversal) {
return nil, err
}
for registry, conf := range configs {
registryHostname := convertToHostname(registry)
if _, ok := res[registryHostname]; !ok {
res[registryHostname] = RegistryInfo{
Source: source,
AuthConfig: conf,
}
}
}
}
return res, nil
}
func getUserConfiguration(dockerAuthConfig string) (string, map[string]types.AuthConfig, error) {
authConfigs, err := readConfigsFromReader(bytes.NewBufferString(dockerAuthConfig))
if errors.Is(err, errPathTraversal) {
return "", nil, err
}
if authConfigs == nil {
return "", nil, nil
}
return authConfigSourceNameUserVariable, authConfigs, nil
}
func getHomeDirConfiguration(username string) (string, map[string]types.AuthConfig, error) {
sourceFile, authConfigs, err := readDockerConfigsFromHomeDir(username)
if errors.Is(err, errPathTraversal) {
return "", nil, err
}
if authConfigs == nil {
return "", nil, nil
}
return sourceFile, authConfigs, nil
}
// EncodeConfig constructs a token from an AuthConfig, suitable for
// authorizing against the Docker API with.
func EncodeConfig(authConfig *types.AuthConfig) (string, error) {
if authConfig == nil {
return "", nil
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(authConfig); err != nil {
return "", err
}
return base64.URLEncoding.EncodeToString(buf.Bytes()), nil
}
func getBuildConfiguration(credentials []common.Credentials) (string, map[string]types.AuthConfig, error) {
authConfigs := make(map[string]types.AuthConfig)
for _, credentials := range credentials {
if credentials.Type != "registry" {
continue
}
authConfigs[credentials.URL] = types.AuthConfig{
Username: credentials.Username,
Password: credentials.Password,
ServerAddress: credentials.URL,
}
}
return authConfigSourceNameJobPayload, authConfigs, nil
}
// splitDockerImageName breaks a reposName into an index name and remote name
func splitDockerImageName(reposName string) (string, string) {
nameParts := strings.SplitN(reposName, "/", 2)
var indexName, remoteName string
if len(nameParts) == 1 || (!strings.Contains(nameParts[0], ".") &&
!strings.Contains(nameParts[0], ":") && nameParts[0] != "localhost") {
// This is a Docker Index repos (ex: samalba/hipache or ubuntu)
// 'docker.io'
indexName = DefaultDockerRegistry
remoteName = reposName
} else {
indexName = nameParts[0]
remoteName = nameParts[1]
}
if indexName == "index."+DefaultDockerRegistry {
indexName = DefaultDockerRegistry
}
return indexName, remoteName
}
// readDockerConfigsFromHomeDir reads known docker config from home
// directory. If no username is provided it will get the home directory for the
// current user.
func readDockerConfigsFromHomeDir(userName string) (string, map[string]types.AuthConfig, error) {
homeDir := HomeDirectory
if userName != "" {
u, err := user.Lookup(userName)
if err != nil {
return "", nil, err
}
homeDir = u.HomeDir
}
if homeDir == "" {
return "", nil, errNoHomeDir
}
configFile := filepath.Join(homeDir, ".docker", "config.json")
r, err := os.Open(configFile)
if err != nil {
configFile = filepath.Join(homeDir, ".dockercfg")
r, err = os.Open(configFile)
if err != nil && !os.IsNotExist(err) {
return "", nil, err
}
}
defer r.Close()
if r == nil {
return "", make(map[string]types.AuthConfig), nil
}
authConfigs, err := readConfigsFromReader(r)
return configFile, authConfigs, err
}
func readConfigsFromReader(r io.Reader) (map[string]types.AuthConfig, error) {
config := &configfile.ConfigFile{}
if err := config.LoadFromReader(r); err != nil {
if errors.Is(err, io.EOF) {
err = nil
}
return nil, err
}
auths := make(map[string]types.AuthConfig)
addAll(auths, config.AuthConfigs)
if config.CredentialsStore != "" {
authsFromCredentialsStore, err := readConfigsFromCredentialsStore(config)
if err != nil {
return nil, err
}
addAll(auths, authsFromCredentialsStore)
}
if config.CredentialHelpers != nil {
authsFromCredentialsHelpers, err := readConfigsFromCredentialsHelper(config)
if err != nil {
return nil, err
}
addAll(auths, authsFromCredentialsHelpers)
}
return auths, nil
}
func readConfigsFromCredentialsStore(config *configfile.ConfigFile) (map[string]types.AuthConfig, error) {
if config.CredentialsStore != filepath.Base(config.CredentialsStore) {
// Fail processing if credential store attempting path traversal are detected
return nil, errPathTraversal
}
store := credentials.NewNativeStore(config, config.CredentialsStore)
newAuths, err := store.GetAll()
if err != nil {
return nil, err
}
return newAuths, nil
}
func readConfigsFromCredentialsHelper(config *configfile.ConfigFile) (map[string]types.AuthConfig, error) {
helpersAuths := make(map[string]types.AuthConfig)
for registry, helper := range config.CredentialHelpers {
if helper != filepath.Base(helper) {
// Fail processing if credential helpers attempting path traversal are detected
return nil, errPathTraversal
}
store := credentials.NewNativeStore(config, helper)
newAuths, err := store.Get(registry)
if err != nil {
return nil, err
}
helpersAuths[registry] = newAuths
}
return helpersAuths, nil
}
func addAll(to, from map[string]types.AuthConfig) {
for reg, ac := range from {
to[reg] = ac
}
}
func convertToHostname(url string) string {
url = strings.ToLower(url)
url = strings.TrimPrefix(url, "http://")
url = strings.TrimPrefix(url, "https://")
nameParts := strings.SplitN(url, "/", 2)
url = nameParts[0]
if url == "index."+DefaultDockerRegistry {
return DefaultDockerRegistry
}
return url
}
package docker
import (
"os"
"strconv"
)
//nolint:lll
type Credentials struct {
Host string `toml:"host,omitempty" json:"host" long:"host" env:"DOCKER_HOST" description:"Docker daemon address"`
CertPath string `toml:"tls_cert_path,omitempty" json:"tls_cert_path" long:"cert-path" env:"DOCKER_CERT_PATH" description:"Certificate path"`
TLSVerify bool `toml:"tls_verify,omitzero" json:"tls_verify" long:"tlsverify" env:"DOCKER_TLS_VERIFY" description:"Use TLS and verify the remote"`
}
func credentialsFromEnv() Credentials {
tlsVerify, _ := strconv.ParseBool(os.Getenv("DOCKER_TLS_VERIFY"))
return Credentials{
Host: os.Getenv("DOCKER_HOST"),
CertPath: os.Getenv("DOCKER_CERT_PATH"),
TLSVerify: tlsVerify,
}
}
package errors
import (
"fmt"
)
// ErrOSNotSupported is used when docker does not support the detected OSType.
// NewErrOSNotSupported is used to initialize this type.
type ErrOSNotSupported struct {
detectedOSType string
}
func (e *ErrOSNotSupported) Error() string {
return fmt.Sprintf("unsupported OSType %q", e.detectedOSType)
}
func (e *ErrOSNotSupported) Is(err error) bool {
_, ok := err.(*ErrOSNotSupported)
return ok
}
// NewErrOSNotSupported creates a ErrOSNotSupported for the specified OSType.
func NewErrOSNotSupported(osType string) *ErrOSNotSupported {
return &ErrOSNotSupported{
detectedOSType: osType,
}
}
package docker
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
"github.com/docker/machine/commands/mcndirs"
"github.com/sirupsen/logrus"
)
const (
defaultDockerMachineExecutable = "docker-machine"
crashreportTokenOption = "--bugsnag-api-token"
crashreportToken = "no-report"
)
var dockerMachineExecutable = defaultDockerMachineExecutable
type logWriter struct {
log func(args ...interface{})
reader *bufio.Reader
}
func (l *logWriter) write(line string) {
line = strings.TrimRight(line, "\n")
if line == "" {
return
}
l.log(line)
}
func (l *logWriter) watch() {
var err error
for err != io.EOF {
var line string
line, err = l.reader.ReadString('\n')
if err != nil && err != io.EOF {
if !strings.Contains(err.Error(), "bad file descriptor") {
logrus.WithError(err).Warn("Problem while reading command output")
}
return
}
l.write(line)
}
}
func newLogWriter(logFunction func(args ...interface{}), reader io.Reader) {
writer := &logWriter{
log: logFunction,
reader: bufio.NewReader(reader),
}
go writer.watch()
}
func stdoutLogWriter(cmd *exec.Cmd, fields logrus.Fields) {
log := logrus.WithFields(fields)
reader, err := cmd.StdoutPipe()
if err == nil {
newLogWriter(log.Infoln, reader)
}
}
func stderrLogWriter(cmd *exec.Cmd, fields logrus.Fields) {
log := logrus.WithFields(fields)
reader, err := cmd.StderrPipe()
if err == nil {
newLogWriter(log.Errorln, reader)
}
}
type machineCommand struct {
cache map[string]machineInfo
cacheLock sync.RWMutex
}
type machineInfo struct {
expires time.Time
canConnect bool
}
func (m *machineCommand) Create(driver, name string, opts ...string) error {
args := []string{
"create",
"--driver", driver,
}
for _, opt := range opts {
args = append(args, "--"+opt)
}
args = append(args, name)
cmd := newDockerMachineCommand(args...)
fields := logrus.Fields{
"operation": "create",
"driver": driver,
"name": name,
}
stdoutLogWriter(cmd, fields)
stderrLogWriter(cmd, fields)
logrus.Debugln("Executing", cmd.Path, cmd.Args)
return cmd.Run()
}
func (m *machineCommand) Provision(name string) error {
cmd := newDockerMachineCommand("provision", name)
fields := logrus.Fields{
"operation": "provision",
"name": name,
}
stdoutLogWriter(cmd, fields)
stderrLogWriter(cmd, fields)
return cmd.Run()
}
func (m *machineCommand) Stop(name string, timeout time.Duration) error {
ctx, ctxCancelFn := context.WithTimeout(context.Background(), timeout)
defer ctxCancelFn()
cmd := newDockerMachineCommandCtx(ctx, "stop", name)
fields := logrus.Fields{
"operation": "stop",
"name": name,
}
stdoutLogWriter(cmd, fields)
stderrLogWriter(cmd, fields)
return cmd.Run()
}
func (m *machineCommand) Remove(name string) error {
cmd := newDockerMachineCommand("rm", "-y", name)
fields := logrus.Fields{
"operation": "remove",
"name": name,
}
stdoutLogWriter(cmd, fields)
stderrLogWriter(cmd, fields)
if err := cmd.Run(); err != nil {
return err
}
m.cacheLock.Lock()
delete(m.cache, name)
m.cacheLock.Unlock()
return nil
}
func (m *machineCommand) List() (hostNames []string, err error) {
dir, err := ioutil.ReadDir(mcndirs.GetMachineDir())
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
for _, file := range dir {
if file.IsDir() && !strings.HasPrefix(file.Name(), ".") {
hostNames = append(hostNames, file.Name())
}
}
return
}
func (m *machineCommand) get(args ...string) (out string, err error) {
// Execute docker-machine to fetch IP
cmd := newDockerMachineCommand(args...)
data, err := cmd.Output()
if err != nil {
return
}
// Save the IP
out = strings.TrimSpace(string(data))
if out == "" {
err = fmt.Errorf("failed to get %v", args)
}
return
}
func (m *machineCommand) IP(name string) (string, error) {
return m.get("ip", name)
}
func (m *machineCommand) URL(name string) (string, error) {
return m.get("url", name)
}
func (m *machineCommand) CertPath(name string) (string, error) {
return m.get("inspect", name, "-f", "{{.HostOptions.AuthOptions.StorePath}}")
}
func (m *machineCommand) Status(name string) (string, error) {
return m.get("status", name)
}
func (m *machineCommand) Exist(name string) bool {
configPath := filepath.Join(mcndirs.GetMachineDir(), name, "config.json")
_, err := os.Stat(configPath)
if err != nil {
return false
}
cmd := newDockerMachineCommand("inspect", name)
fields := logrus.Fields{
"operation": "exists",
"name": name,
}
stderrLogWriter(cmd, fields)
return cmd.Run() == nil
}
func (m *machineCommand) CanConnect(name string, skipCache bool) bool {
m.cacheLock.RLock()
cachedInfo, ok := m.cache[name]
m.cacheLock.RUnlock()
if ok && !skipCache && time.Now().Before(cachedInfo.expires) {
return cachedInfo.canConnect
}
canConnect := m.canConnect(name)
if !canConnect {
return false // we only cache positive hits. Machines usually do not disconnect.
}
m.cacheLock.Lock()
m.cache[name] = machineInfo{
expires: time.Now().Add(5 * time.Minute),
canConnect: true,
}
m.cacheLock.Unlock()
return true
}
func (m *machineCommand) canConnect(name string) bool {
// Execute docker-machine config which actively ask the machine if it is up and online
cmd := newDockerMachineCommand("config", name)
err := cmd.Run()
return err == nil
}
func (m *machineCommand) Credentials(name string) (dc Credentials, err error) {
if !m.CanConnect(name, true) {
err = errors.New("can't connect")
return
}
dc.TLSVerify = true
dc.Host, err = m.URL(name)
if err == nil {
dc.CertPath, err = m.CertPath(name)
}
return
}
func newDockerMachineCommandCtx(ctx context.Context, args ...string) *exec.Cmd {
token := os.Getenv("MACHINE_BUGSNAG_API_TOKEN")
if token == "" {
token = crashreportToken
}
commandArgs := []string{
fmt.Sprintf("%s=%s", crashreportTokenOption, token),
}
commandArgs = append(commandArgs, args...)
cmd := exec.CommandContext(ctx, dockerMachineExecutable, commandArgs...)
cmd.Env = os.Environ()
return cmd
}
func newDockerMachineCommand(args ...string) *exec.Cmd {
return newDockerMachineCommandCtx(context.Background(), args...)
}
func NewMachineCommand() Machine {
return &machineCommand{
cache: map[string]machineInfo{},
}
}
package docker
import (
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"path/filepath"
"runtime"
"time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/api/types/volume"
"github.com/docker/docker/client"
"github.com/docker/docker/pkg/jsonmessage"
"github.com/docker/go-connections/tlsconfig"
"github.com/sirupsen/logrus"
)
// The default API version used to create a new docker client.
const DefaultAPIVersion = "1.25"
// ErrRedirectNotAllowed is returned when we get a 3xx request from the Docker
// client to prevent any redirections to malicious docker clients.
var ErrRedirectNotAllowed = errors.New("redirects disallowed")
// IsErrNotFound checks whether a returned error is due to an image or container
// not being found. Proxies the docker implementation.
func IsErrNotFound(err error) bool {
unwrapped := errors.Unwrap(err)
if unwrapped != nil {
err = unwrapped
}
return client.IsErrNotFound(err)
}
// type officialDockerClient wraps a "github.com/docker/docker/client".Client,
// giving it the methods it needs to satisfy the docker.Client interface
type officialDockerClient struct {
client *client.Client
// Close() means "close idle connections held by engine-api's transport"
Transport *http.Transport
}
func newOfficialDockerClient(c Credentials, apiVersion string) (*officialDockerClient, error) {
transport, err := newHTTPTransport(c)
if err != nil {
logrus.Errorln("Error creating TLS Docker client:", err)
return nil, err
}
httpClient := &http.Client{
Transport: transport,
CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
return ErrRedirectNotAllowed
},
}
dockerClient, err := client.NewClientWithOpts(
client.WithHost(c.Host),
client.WithVersion(apiVersion),
client.WithHTTPClient(httpClient),
)
if err != nil {
transport.CloseIdleConnections()
logrus.Errorln("Error creating Docker client:", err)
return nil, err
}
return &officialDockerClient{
client: dockerClient,
Transport: transport,
}, nil
}
func wrapError(method string, err error, started time.Time) error {
if err == nil {
return nil
}
seconds := int(time.Since(started).Seconds())
if _, file, line, ok := runtime.Caller(2); ok {
return fmt.Errorf("%w (%s:%d:%ds)", err, filepath.Base(file), line, seconds)
}
return fmt.Errorf("%w (%s:%ds)", err, method, seconds)
}
func (c *officialDockerClient) ImageInspectWithRaw(
ctx context.Context,
imageID string,
) (types.ImageInspect, []byte, error) {
started := time.Now()
image, data, err := c.client.ImageInspectWithRaw(ctx, imageID)
return image, data, wrapError("ImageInspectWithRaw", err, started)
}
func (c *officialDockerClient) ContainerList(
ctx context.Context,
options types.ContainerListOptions,
) ([]types.Container, error) {
started := time.Now()
containers, err := c.client.ContainerList(ctx, options)
return containers, wrapError("ContainerList", err, started)
}
func (c *officialDockerClient) ContainerCreate(
ctx context.Context,
config *container.Config,
hostConfig *container.HostConfig,
networkingConfig *network.NetworkingConfig,
containerName string,
) (container.ContainerCreateCreatedBody, error) {
started := time.Now()
container, err := c.client.ContainerCreate(ctx, config, hostConfig, networkingConfig, nil, containerName)
return container, wrapError("ContainerCreate", err, started)
}
func (c *officialDockerClient) ContainerStart(
ctx context.Context,
containerID string,
options types.ContainerStartOptions,
) error {
started := time.Now()
err := c.client.ContainerStart(ctx, containerID, options)
return wrapError("ContainerCreate", err, started)
}
func (c *officialDockerClient) ContainerKill(ctx context.Context, containerID string, signal string) error {
started := time.Now()
err := c.client.ContainerKill(ctx, containerID, signal)
return wrapError("ContainerKill", err, started)
}
func (c *officialDockerClient) ContainerInspect(ctx context.Context, containerID string) (types.ContainerJSON, error) {
started := time.Now()
data, err := c.client.ContainerInspect(ctx, containerID)
return data, wrapError("ContainerInspect", err, started)
}
func (c *officialDockerClient) ContainerAttach(
ctx context.Context,
container string,
options types.ContainerAttachOptions,
) (types.HijackedResponse, error) {
started := time.Now()
response, err := c.client.ContainerAttach(ctx, container, options)
return response, wrapError("ContainerAttach", err, started)
}
func (c *officialDockerClient) ContainerRemove(
ctx context.Context,
containerID string,
options types.ContainerRemoveOptions,
) error {
started := time.Now()
err := c.client.ContainerRemove(ctx, containerID, options)
return wrapError("ContainerRemove", err, started)
}
func (c *officialDockerClient) ContainerWait(
ctx context.Context,
containerID string,
condition container.WaitCondition,
) (<-chan container.ContainerWaitOKBody, <-chan error) {
return c.client.ContainerWait(ctx, containerID, condition)
}
func (c *officialDockerClient) ContainerLogs(
ctx context.Context,
container string,
options types.ContainerLogsOptions,
) (io.ReadCloser, error) {
started := time.Now()
rc, err := c.client.ContainerLogs(ctx, container, options)
return rc, wrapError("ContainerLogs", err, started)
}
func (c *officialDockerClient) ContainerExecCreate(
ctx context.Context,
container string,
config types.ExecConfig,
) (types.IDResponse, error) {
started := time.Now()
resp, err := c.client.ContainerExecCreate(ctx, container, config)
return resp, wrapError("ContainerExecCreate", err, started)
}
func (c *officialDockerClient) ContainerExecAttach(
ctx context.Context,
execID string,
config types.ExecStartCheck,
) (types.HijackedResponse, error) {
started := time.Now()
resp, err := c.client.ContainerExecAttach(ctx, execID, config)
return resp, wrapError("ContainerExecAttach", err, started)
}
func (c *officialDockerClient) NetworkCreate(
ctx context.Context,
networkName string,
options types.NetworkCreate,
) (types.NetworkCreateResponse, error) {
started := time.Now()
response, err := c.client.NetworkCreate(ctx, networkName, options)
return response, wrapError("NetworkCreate", err, started)
}
func (c *officialDockerClient) NetworkRemove(ctx context.Context, networkID string) error {
started := time.Now()
err := c.client.NetworkRemove(ctx, networkID)
return wrapError("NetworkRemove", err, started)
}
func (c *officialDockerClient) NetworkDisconnect(ctx context.Context, networkID, containerID string, force bool) error {
started := time.Now()
err := c.client.NetworkDisconnect(ctx, networkID, containerID, force)
return wrapError("NetworkDisconnect", err, started)
}
func (c *officialDockerClient) NetworkList(
ctx context.Context,
options types.NetworkListOptions,
) ([]types.NetworkResource, error) {
started := time.Now()
networks, err := c.client.NetworkList(ctx, options)
return networks, wrapError("NetworkList", err, started)
}
func (c *officialDockerClient) NetworkInspect(ctx context.Context, networkID string) (types.NetworkResource, error) {
started := time.Now()
resource, err := c.client.NetworkInspect(ctx, networkID, types.NetworkInspectOptions{})
return resource, wrapError("NetworkInspect", err, started)
}
func (c *officialDockerClient) VolumeCreate(
ctx context.Context,
options volume.VolumeCreateBody,
) (types.Volume, error) {
started := time.Now()
v, err := c.client.VolumeCreate(ctx, options)
return v, wrapError("VolumeCreate", err, started)
}
func (c *officialDockerClient) VolumeRemove(ctx context.Context, volumeID string, force bool) error {
started := time.Now()
err := c.client.VolumeRemove(ctx, volumeID, force)
return wrapError("VolumeRemove", err, started)
}
func (c *officialDockerClient) VolumeInspect(ctx context.Context, volumeID string) (types.Volume, error) {
started := time.Now()
v, err := c.client.VolumeInspect(ctx, volumeID)
return v, wrapError("VolumeInspect", err, started)
}
func (c *officialDockerClient) Info(ctx context.Context) (types.Info, error) {
started := time.Now()
info, err := c.client.Info(ctx)
return info, wrapError("Info", err, started)
}
func (c *officialDockerClient) ImageImportBlocking(
ctx context.Context,
source types.ImageImportSource,
ref string,
options types.ImageImportOptions,
) error {
started := time.Now()
rc, err := c.client.ImageImport(ctx, source, ref, options)
if err != nil {
return wrapError("ImageImport", err, started)
}
return wrapError("ImageImport", c.handleEventStream(rc), started)
}
func (c *officialDockerClient) ImagePullBlocking(
ctx context.Context,
ref string,
options types.ImagePullOptions,
) error {
started := time.Now()
rc, err := c.client.ImagePull(ctx, ref, options)
if err != nil {
return wrapError("ImagePull", err, started)
}
return wrapError("ImagePull", c.handleEventStream(rc), started)
}
func (c *officialDockerClient) handleEventStream(rc io.ReadCloser) error {
defer func() { _ = rc.Close() }()
return jsonmessage.DisplayJSONMessagesStream(rc, ioutil.Discard, 0, false, nil)
}
func (c *officialDockerClient) Close() error {
c.Transport.CloseIdleConnections()
return nil
}
// New attempts to create a new Docker client of the specified version. If the
// specified version is empty, it will use the default version.
//
// If no host is given in the Credentials, it will attempt to look up
// details from the environment. If that fails, it will use the default
// connection details for your platform.
func New(c Credentials, apiVersion string) (Client, error) {
if c.Host == "" {
c = credentialsFromEnv()
}
// Use the default if nothing is specified by caller *or* environment
if c.Host == "" {
c.Host = client.DefaultDockerHost
}
if apiVersion == "" {
apiVersion = DefaultAPIVersion
}
return newOfficialDockerClient(c, apiVersion)
}
func newHTTPTransport(c Credentials) (*http.Transport, error) {
url, err := client.ParseHostURL(c.Host)
if err != nil {
return nil, err
}
tr := &http.Transport{}
if err := configureTransport(tr, url.Scheme, url.Host); err != nil {
return nil, err
}
// FIXME: is a TLS connection with InsecureSkipVerify == true ever wanted?
if c.TLSVerify {
options := tlsconfig.Options{}
if c.CertPath != "" {
options.CAFile = filepath.Join(c.CertPath, "ca.pem")
options.CertFile = filepath.Join(c.CertPath, "cert.pem")
options.KeyFile = filepath.Join(c.CertPath, "key.pem")
}
tlsConfig, err := tlsconfig.Client(options)
if err != nil {
tr.CloseIdleConnections()
return nil, err
}
tr.TLSClientConfig = tlsConfig
}
return tr, nil
}
package docker
import (
"net"
"net/http"
"time"
"github.com/docker/go-connections/sockets"
)
const defaultTimeout = 300 * time.Second
const defaultKeepAlive = 10 * time.Second
const defaultTLSHandshakeTimeout = 60 * time.Second
const defaultResponseHeaderTimeout = 120 * time.Second
const defaultExpectContinueTimeout = 120 * time.Second
const defaultIdleConnTimeout = 10 * time.Second
// configureTransport configures the specified Transport according to the
// specified proto and addr.
// If the proto is unix (using a unix socket to communicate) or npipe the
// compression is disabled.
func configureTransport(tr *http.Transport, proto, addr string) error {
err := sockets.ConfigureTransport(tr, proto, addr)
if err != nil {
return err
}
tr.TLSHandshakeTimeout = defaultTLSHandshakeTimeout
tr.ResponseHeaderTimeout = defaultResponseHeaderTimeout
tr.ExpectContinueTimeout = defaultExpectContinueTimeout
tr.IdleConnTimeout = defaultIdleConnTimeout
// for network protocols set custom sockets with keep-alive
if proto == "tcp" || proto == "http" || proto == "https" {
dialer, err := sockets.DialerFromEnvironment(&net.Dialer{
Timeout: defaultTimeout,
KeepAlive: defaultKeepAlive,
})
if err != nil {
return err
}
tr.Dial = dialer.Dial // nolint:staticcheck
}
return nil
}
package test
// NotFoundError implements the interface that docker client checks for
// `IsErrNotFound`
// https://github.com/moby/moby/blob/f6a5ccf492e8eab969ffad8404117806b4a15a35/client/errors.go#L36-L49
type NotFoundError struct {
}
func (e *NotFoundError) NotFound() bool {
return true
}
func (e *NotFoundError) Error() string {
return "not found"
}
package helpers
import (
"fmt"
"io"
"github.com/sirupsen/logrus"
)
type fatalLogHook struct {
output io.Writer
}
func (s *fatalLogHook) Levels() []logrus.Level {
return []logrus.Level{
logrus.FatalLevel,
}
}
func (s *fatalLogHook) Fire(e *logrus.Entry) error {
_, _ = fmt.Fprintln(s.output, e.Message)
panic(e)
}
func MakeFatalToPanic() func() {
logger := logrus.StandardLogger()
hooks := make(logrus.LevelHooks)
hooks.Add(&fatalLogHook{output: logger.Out})
oldHooks := logger.ReplaceHooks(hooks)
return func() {
logger.ReplaceHooks(oldHooks)
}
}
package featureflags
import (
"strconv"
"github.com/sirupsen/logrus"
)
const (
CmdDisableDelayedErrorLevelExpansion string = "FF_CMD_DISABLE_DELAYED_ERROR_LEVEL_EXPANSION"
NetworkPerBuild string = "FF_NETWORK_PER_BUILD"
UseLegacyKubernetesExecutionStrategy string = "FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY"
UseDirectDownload string = "FF_USE_DIRECT_DOWNLOAD"
SkipNoOpBuildStages string = "FF_SKIP_NOOP_BUILD_STAGES"
UseFastzip string = "FF_USE_FASTZIP"
GitLabRegistryHelperImage string = "FF_GITLAB_REGISTRY_HELPER_IMAGE"
DisableUmaskForDockerExecutor string = "FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR"
EnableBashExitCodeCheck string = "FF_ENABLE_BASH_EXIT_CODE_CHECK"
UseWindowsLegacyProcessStrategy string = "FF_USE_WINDOWS_LEGACY_PROCESS_STRATEGY"
SkipDockerMachineProvisionOnCreationFailure string = "FF_SKIP_DOCKER_MACHINE_PROVISION_ON_CREATION_FAILURE"
UseNewEvalStrategy string = "FF_USE_NEW_BASH_EVAL_STRATEGY"
UsePowershellPathResolver string = "FF_USE_POWERSHELL_PATH_RESOLVER"
)
type FeatureFlag struct {
Name string
DefaultValue bool
Deprecated bool
ToBeRemovedWith string
Description string
}
// REMEMBER to update the documentation after adding or removing a feature flag
//
// Please use `make update_feature_flags_docs` to make the update automatic and
// properly formatted. It will replace the existing table with the new one, computed
// basing on the values below
var flags = []FeatureFlag{
{
Name: CmdDisableDelayedErrorLevelExpansion,
DefaultValue: false,
Deprecated: false,
ToBeRemovedWith: "",
Description: "Disables [EnableDelayedExpansion](https://ss64.com/nt/delayedexpansion.html) for " +
"error checking for when using [Window Batch](../shells/index.md#windows-batch) shell",
},
{
Name: NetworkPerBuild,
DefaultValue: false,
Deprecated: false,
ToBeRemovedWith: "",
Description: "Enables creation of a Docker [network per build](../executors/docker.md#networking) with " +
"the `docker` executor",
},
{
Name: UseLegacyKubernetesExecutionStrategy,
DefaultValue: false,
Deprecated: false,
ToBeRemovedWith: "",
Description: "When set to `false` disables execution of remote Kubernetes commands through `exec` in " +
"favor of `attach` to solve problems like " +
"[#4119](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4119)",
},
{
Name: UseDirectDownload,
DefaultValue: true,
Deprecated: false,
ToBeRemovedWith: "",
Description: "When set to `true` Runner tries to direct-download all artifacts instead of proxying " +
"through GitLab on a first try. Enabling might result in a download failures due to problem validating " +
"TLS certificate of Object Storage if it is enabled by GitLab. " +
"See [Self-signed certificates or custom Certification Authorities](tls-self-signed.md)",
},
{
Name: SkipNoOpBuildStages,
DefaultValue: true,
Deprecated: false,
ToBeRemovedWith: "",
Description: "When set to `false` all build stages are executed even if running them has no effect",
},
{
Name: UseFastzip,
DefaultValue: false,
Deprecated: false,
ToBeRemovedWith: "",
Description: "Fastzip is a performant archiver for cache/artifact archiving and extraction",
},
{
Name: GitLabRegistryHelperImage,
DefaultValue: true,
Deprecated: false,
ToBeRemovedWith: "",
Description: "Use GitLab Runner helper image for the Docker and " +
"Kubernetes executors from `registry.gitlab.com` instead of Docker Hub",
},
{
Name: DisableUmaskForDockerExecutor,
DefaultValue: false,
Deprecated: false,
ToBeRemovedWith: "",
Description: "If enabled will remove the usage of `umask 0000` call for jobs executed with `docker` " +
"executor. Instead Runner will try to discover the UID and GID of the user configured for the image used " +
"by the build container and will change the ownership of the working directory and files by running the " +
"`chmod` command in the predefined container (after updating sources, restoring cache and " +
"downloading artifacts). POSIX utility `id` must be installed and operational in the build image " +
"for this feature flag. Runner will execute `id` with options `-u` and `-g` to retrieve the UID and GID.",
},
{
Name: EnableBashExitCodeCheck,
DefaultValue: false,
Deprecated: false,
ToBeRemovedWith: "",
Description: "If enabled, bash scripts don't rely solely on `set -e`, but check for a non-zero exit code " +
"after each script command is executed.",
},
{
Name: UseWindowsLegacyProcessStrategy,
DefaultValue: true,
Deprecated: false,
ToBeRemovedWith: "",
Description: "When disabled, processes that Runner creates on Windows (shell and custom executor) will be " +
"created with additional setup that should improve process termination. This is currently experimental " +
"and how we setup these processes may change as we continue to improve this. When set to `true`, legacy " +
"process setup is used. To successfully and gracefully drain a Windows Runner, this feature flag should" +
"be set to `false`.",
},
{
Name: SkipDockerMachineProvisionOnCreationFailure,
DefaultValue: true,
Deprecated: false,
ToBeRemovedWith: "",
Description: "With the `docker+machine` executor, when a machine is " +
"not created, `docker-machine provision` runs for X amount of times. When " +
"this feature flag is set to `true`, it skips `docker-machine provision`, " +
"removes the machine, and creates another machine instead.",
},
{
Name: UseNewEvalStrategy,
DefaultValue: false,
Deprecated: false,
ToBeRemovedWith: "",
Description: "When set to `true`, the Bash `eval` call is executed in a subshell to help with proper exit " +
"code detection of the script executed.",
},
{
Name: UsePowershellPathResolver,
DefaultValue: false,
Deprecated: false,
ToBeRemovedWith: "",
Description: "When enabled, Powershell resolves pathnames rather than Runner using OS-specific filepath " +
"functions that are specific to where Runner is hosted.",
},
}
func GetAll() []FeatureFlag {
return flags
}
func IsOn(logger logrus.FieldLogger, value string) bool {
if value == "" {
return false
}
on, err := strconv.ParseBool(value)
if err != nil {
logger.WithError(err).
WithField("value", value).
Error("Error while parsing the value of feature flag")
return false
}
return on
}
package gitlab_ci_yaml_parser
import (
"encoding/json"
"fmt"
"strings"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
)
type DataBag map[string]interface{}
func (m *DataBag) Get(keys ...string) (interface{}, bool) {
return helpers.GetMapKey(*m, keys...)
}
func (m *DataBag) GetSlice(keys ...string) ([]interface{}, bool) {
slice, ok := helpers.GetMapKey(*m, keys...)
if slice != nil {
return slice.([]interface{}), ok
}
return nil, false
}
func (m *DataBag) GetStringSlice(keys ...string) (slice []string, ok bool) {
rawSlice, ok := m.GetSlice(keys...)
if !ok {
return
}
for _, rawElement := range rawSlice {
if element, ok := rawElement.(string); ok {
slice = append(slice, element)
}
}
return
}
func (m *DataBag) GetSubOptions(keys ...string) (result DataBag, ok bool) {
value, ok := helpers.GetMapKey(*m, keys...)
if ok {
result, ok = value.(map[string]interface{})
}
return
}
func (m *DataBag) GetString(keys ...string) (result string, ok bool) {
value, ok := helpers.GetMapKey(*m, keys...)
if ok {
result, ok = value.(string)
}
return
}
func (m *DataBag) Decode(result interface{}, keys ...string) error {
value, ok := m.Get(keys...)
if !ok {
return fmt.Errorf("key not found %v", strings.Join(keys, "."))
}
data, err := json.Marshal(value)
if err != nil {
return err
}
return json.Unmarshal(data, result)
}
func convertMapToStringMap(in interface{}) (out interface{}, err error) {
mapString := make(map[string]interface{})
switch convMap := in.(type) {
case map[string]interface{}:
mapString = convMap
case map[interface{}]interface{}:
for k, v := range convMap {
key, ok := k.(string)
if !ok {
return nil, fmt.Errorf("failed to convert %v to string", k)
}
mapString[key] = v
}
default:
return in, nil
}
for k, v := range mapString {
mapString[k], err = convertMapToStringMap(v)
if err != nil {
return
}
}
return mapString, nil
}
func (m *DataBag) Sanitize() (err error) {
n := make(DataBag)
for k, v := range *m {
n[k], err = convertMapToStringMap(v)
if err != nil {
return
}
}
*m = n
return
}
func getOptionsMap(optionKey string, primary, secondary DataBag) (value DataBag) {
value, ok := primary.GetSubOptions(optionKey)
if !ok {
value, _ = secondary.GetSubOptions(optionKey)
}
return
}
func getOptions(optionKey string, primary, secondary DataBag) (value []interface{}, ok bool) {
value, ok = primary.GetSlice(optionKey)
if !ok {
value, ok = secondary.GetSlice(optionKey)
}
return
}
package gitlab_ci_yaml_parser
import (
"errors"
"fmt"
"io/ioutil"
"strconv"
"strings"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gopkg.in/yaml.v2"
)
type GitLabCiYamlParser struct {
filename string
jobName string
config DataBag
jobConfig DataBag
}
func (c *GitLabCiYamlParser) parseFile() (err error) {
data, err := ioutil.ReadFile(c.filename)
if err != nil {
return err
}
config := make(DataBag)
err = yaml.Unmarshal(data, config)
if err != nil {
return err
}
err = config.Sanitize()
if err != nil {
return err
}
c.config = config
return
}
func (c *GitLabCiYamlParser) loadJob() (err error) {
jobConfig, ok := c.config.GetSubOptions(c.jobName)
if !ok {
return fmt.Errorf("no job named %q", c.jobName)
}
c.jobConfig = jobConfig
return
}
func (c *GitLabCiYamlParser) prepareJobInfo(job *common.JobResponse) (err error) {
job.JobInfo = common.JobInfo{
Name: c.jobName,
}
if stage, ok := c.jobConfig.GetString("stage"); ok {
job.JobInfo.Stage = stage
} else {
job.JobInfo.Stage = "test"
}
return
}
func (c *GitLabCiYamlParser) getCommands(commands interface{}) (common.StepScript, error) {
switch t := commands.(type) {
case []interface{}:
var steps common.StepScript
for _, line := range t {
if lineText, ok := line.(string); ok {
steps = append(steps, lineText)
} else {
return common.StepScript{}, errors.New("unsupported script")
}
}
return steps, nil
case string:
return strings.Split(t, "\n"), nil
default:
if commands != nil {
return common.StepScript{}, errors.New("unsupported script")
}
}
return common.StepScript{}, nil
}
func (c *GitLabCiYamlParser) prepareSteps(job *common.JobResponse) error {
if c.jobConfig["script"] == nil {
return fmt.Errorf("missing 'script' for job")
}
var scriptCommands, afterScriptCommands common.StepScript
// get before_script
beforeScript, err := c.getCommands(c.config["before_script"])
if err != nil {
return err
}
// get job before_script
jobBeforeScript, err := c.getCommands(c.jobConfig["before_script"])
if err != nil {
return err
}
if len(jobBeforeScript) < 1 {
scriptCommands = beforeScript
} else {
scriptCommands = jobBeforeScript
}
// get script
script, err := c.getCommands(c.jobConfig["script"])
if err != nil {
return err
}
scriptCommands = append(scriptCommands, script...)
afterScriptCommands, err = c.getCommands(c.jobConfig["after_script"])
if err != nil {
return err
}
job.Steps = common.Steps{
common.Step{
Name: common.StepNameScript,
Script: scriptCommands,
Timeout: 3600,
When: common.StepWhenOnSuccess,
AllowFailure: false,
},
common.Step{
Name: common.StepNameAfterScript,
Script: afterScriptCommands,
Timeout: 3600,
When: common.StepWhenAlways,
AllowFailure: false,
},
}
return nil
}
func (c *GitLabCiYamlParser) buildDefaultVariables(job *common.JobResponse) common.JobVariables {
return common.JobVariables{
{Key: "CI", Value: "true", Public: true, Internal: true, File: false},
{Key: "GITLAB_CI", Value: "true", Public: true, Internal: true, File: false},
{Key: "CI_SERVER_NAME", Value: "GitLab CI", Public: true, Internal: true, File: false},
{Key: "CI_SERVER_VERSION", Value: "", Public: true, Internal: true, File: false},
{Key: "CI_SERVER_REVISION", Value: "", Public: true, Internal: true, File: false},
{Key: "CI_PROJECT_ID", Value: strconv.Itoa(job.JobInfo.ProjectID), Public: true, Internal: true, File: false},
{Key: "CI_JOB_ID", Value: strconv.Itoa(job.ID), Public: true, Internal: true, File: false},
{Key: "CI_JOB_NAME", Value: job.JobInfo.Name, Public: true, Internal: true, File: false},
{Key: "CI_JOB_STAGE", Value: job.JobInfo.Stage, Public: true, Internal: true, File: false},
{Key: "CI_JOB_TOKEN", Value: job.Token, Public: true, Internal: true, File: false},
{Key: "CI_REPOSITORY_URL", Value: job.GitInfo.RepoURL, Public: true, Internal: true, File: false},
{Key: "CI_COMMIT_SHA", Value: job.GitInfo.Sha, Public: true, Internal: true, File: false},
{Key: "CI_COMMIT_BEFORE_SHA", Value: job.GitInfo.BeforeSha, Public: true, Internal: true, File: false},
{Key: "CI_COMMIT_REF_NAME", Value: job.GitInfo.Ref, Public: true, Internal: true, File: false},
}
}
func (c *GitLabCiYamlParser) buildVariables(
configVariables interface{},
) (buildVariables common.JobVariables, err error) {
if variables, ok := configVariables.(map[string]interface{}); ok {
for key, value := range variables {
if valueText, ok := value.(string); ok {
buildVariables = append(buildVariables, common.JobVariable{
Key: key,
Value: valueText,
Public: true,
})
} else {
err = fmt.Errorf("invalid value for variable %q", key)
}
}
} else if configVariables != nil {
err = errors.New("unsupported variables")
}
return
}
func (c *GitLabCiYamlParser) prepareVariables(job *common.JobResponse) error {
job.Variables = common.JobVariables{}
defaultVariables := c.buildDefaultVariables(job)
job.Variables = append(job.Variables, defaultVariables...)
globalVariables, err := c.buildVariables(c.config["variables"])
if err != nil {
return err
}
job.Variables = append(job.Variables, globalVariables...)
jobVariables, err := c.buildVariables(c.jobConfig["variables"])
if err != nil {
return err
}
job.Variables = append(job.Variables, jobVariables...)
return nil
}
func (c *GitLabCiYamlParser) prepareImage(job *common.JobResponse) error {
job.Image = common.Image{}
if imageName, ok := c.jobConfig.GetString("image"); ok {
job.Image.Name = imageName
return nil
}
if imageDefinition, ok := c.jobConfig.GetSubOptions("image"); ok {
job.Image.Name, _ = imageDefinition.GetString("name")
job.Image.Entrypoint, _ = imageDefinition.GetStringSlice("entrypoint")
return nil
}
if imageName, ok := c.config.GetString("image"); ok {
job.Image.Name = imageName
return nil
}
if imageDefinition, ok := c.config.GetSubOptions("image"); ok {
job.Image.Name, _ = imageDefinition.GetString("name")
job.Image.Entrypoint, _ = imageDefinition.GetStringSlice("entrypoint")
return nil
}
return nil
}
func parseExtendedServiceDefinitionMap(serviceDefinition map[interface{}]interface{}) (image common.Image) {
service := make(DataBag)
for key, value := range serviceDefinition {
service[key.(string)] = value
}
image.Name, _ = service.GetString("name")
image.Alias, _ = service.GetString("alias")
image.Command, _ = service.GetStringSlice("command")
image.Entrypoint, _ = service.GetStringSlice("entrypoint")
return
}
func (c *GitLabCiYamlParser) prepareServices(job *common.JobResponse) (err error) {
job.Services = common.Services{}
if servicesMap, ok := getOptions("services", c.jobConfig, c.config); ok {
for _, service := range servicesMap {
if serviceName, ok := service.(string); ok {
job.Services = append(job.Services, common.Image{
Name: serviceName,
})
continue
}
if serviceDefinition, ok := service.(map[interface{}]interface{}); ok {
job.Services = append(job.Services, parseExtendedServiceDefinitionMap(serviceDefinition))
}
}
}
return
}
func (c *GitLabCiYamlParser) prepareArtifacts(job *common.JobResponse) error {
var ok bool
artifactsMap := getOptionsMap("artifacts", c.jobConfig, c.config)
artifactsPaths, _ := artifactsMap.GetSlice("paths")
paths := common.ArtifactPaths{}
for _, path := range artifactsPaths {
paths = append(paths, path.(string))
}
var artifactsName string
if artifactsName, ok = artifactsMap.GetString("name"); !ok {
artifactsName = ""
}
var artifactsUntracked interface{}
if artifactsUntracked, ok = artifactsMap.Get("untracked"); !ok {
artifactsUntracked = false
}
var artifactsWhen string
if artifactsWhen, ok = artifactsMap.GetString("when"); !ok {
artifactsWhen = string(common.ArtifactWhenOnSuccess)
}
var artifactsExpireIn string
if artifactsExpireIn, ok = artifactsMap.GetString("expireIn"); !ok {
artifactsExpireIn = ""
}
job.Artifacts = make(common.Artifacts, 1)
job.Artifacts[0] = common.Artifact{
Name: artifactsName,
Untracked: artifactsUntracked.(bool),
Paths: paths,
When: common.ArtifactWhen(artifactsWhen),
ExpireIn: artifactsExpireIn,
}
return nil
}
func (c *GitLabCiYamlParser) prepareCache(job *common.JobResponse) error {
var ok bool
cacheMap := getOptionsMap("cache", c.jobConfig, c.config)
cachePaths, _ := cacheMap.GetSlice("paths")
paths := common.ArtifactPaths{}
for _, path := range cachePaths {
paths = append(paths, path.(string))
}
var cacheKey string
if cacheKey, ok = cacheMap.GetString("key"); !ok {
cacheKey = ""
}
var cacheUntracked interface{}
if cacheUntracked, ok = cacheMap.Get("untracked"); !ok {
cacheUntracked = false
}
job.Cache = make(common.Caches, 1)
job.Cache[0] = common.Cache{
Key: cacheKey,
Untracked: cacheUntracked.(bool),
Paths: paths,
}
return nil
}
func (c *GitLabCiYamlParser) ParseYaml(job *common.JobResponse) (err error) {
err = c.parseFile()
if err != nil {
return err
}
err = c.loadJob()
if err != nil {
return err
}
parsers := []struct {
method func(job *common.JobResponse) error
}{
{c.prepareJobInfo},
{c.prepareSteps},
{c.prepareVariables},
{c.prepareImage},
{c.prepareServices},
{c.prepareArtifacts},
{c.prepareCache},
}
for _, parser := range parsers {
err = parser.method(job)
if err != nil {
return err
}
}
return nil
}
func NewGitLabCiYamlParser(jobName string) *GitLabCiYamlParser {
return &GitLabCiYamlParser{
filename: ".gitlab-ci.yml",
jobName: jobName,
}
}
package helpers
import (
"os"
"github.com/docker/docker/pkg/homedir"
)
func GetCurrentWorkingDirectory() string {
dir, err := os.Getwd()
if err == nil {
return dir
}
return ""
}
func GetHomeDir() string {
return homedir.Get()
}
package helpers
import (
"fmt"
"os/exec"
"testing"
"github.com/stretchr/testify/assert"
)
func SkipIntegrationTests(t *testing.T, cmd ...string) {
if testing.Short() {
t.Skip("Skipping long tests")
}
if len(cmd) == 0 {
return
}
executable, err := exec.LookPath(cmd[0])
if err != nil {
t.Skip(cmd[0], "doesn't exist", err)
}
if err := executeCommandSucceeded(executable, cmd[1:]); err != nil {
assert.FailNow(t, "failed integration test command", "%q failed with error: %v", executable, err)
}
}
// executeCommandSucceeded tests whether a particular command execution successfully
// completes. If it does not, it returns the error produced.
func executeCommandSucceeded(executable string, args []string) error {
cmd := exec.Command(executable, args...)
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("%w - %s", err, string(out))
}
return nil
}
package parallels
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"os/exec"
"regexp"
"runtime"
"strconv"
"strings"
"time"
"github.com/sirupsen/logrus"
)
type StatusType string
const (
NotFound StatusType = "notfound"
Invalid StatusType = "invalid"
Stopped StatusType = "stopped"
Suspended StatusType = "suspended"
Running StatusType = "running"
// TODO: more statuses
)
const (
prlctlPath = "prlctl"
dhcpLeases = "/Library/Preferences/Parallels/parallels_dhcp_leases"
)
func PrlctlOutput(args ...string) (string, error) {
if runtime.GOOS != "darwin" {
return "", fmt.Errorf("parallels works only on \"darwin\" platform")
}
var stdout, stderr bytes.Buffer
logrus.Debugf("Executing PrlctlOutput: %#v", args)
cmd := exec.Command(prlctlPath, args...)
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
stderrString := strings.TrimSpace(stderr.String())
if _, ok := err.(*exec.ExitError); ok {
err = fmt.Errorf("calling prlctl: %s", stderrString)
}
return stdout.String(), err
}
func Prlctl(args ...string) error {
_, err := PrlctlOutput(args...)
return err
}
func Exec(vmName string, args ...string) (string, error) {
args2 := append([]string{"exec", vmName}, args...)
return PrlctlOutput(args2...)
}
func Version() (string, error) {
out, err := PrlctlOutput("--version")
if err != nil {
return "", err
}
versionRe := regexp.MustCompile(`prlctl version (\d+\.\d+.\d+)`)
matches := versionRe.FindStringSubmatch(out)
if matches == nil {
return "", fmt.Errorf("could not find Parallels Desktop version in output:\n%s", out)
}
version := matches[1]
logrus.Debugf("Parallels Desktop version: %s", version)
return version, nil
}
func Exist(name string) bool {
err := Prlctl("list", name, "--no-header", "--output", "status")
return err == nil
}
func CreateTemplate(vmName, templateName string) error {
return Prlctl("clone", vmName, "--name", templateName, "--template", "--linked")
}
func CreateOsVM(vmName, templateName string) error {
return Prlctl("create", vmName, "--ostemplate", templateName)
}
func CreateSnapshot(vmName, snapshotName string) error {
return Prlctl("snapshot", vmName, "--name", snapshotName)
}
func GetDefaultSnapshot(vmName string) (string, error) {
output, err := PrlctlOutput("snapshot-list", vmName)
if err != nil {
return "", err
}
lines := strings.Split(output, "\n")
for _, line := range lines {
pos := strings.Index(line, " *")
if pos >= 0 {
snapshot := line[pos+2:]
snapshot = strings.TrimSpace(snapshot)
if len(snapshot) > 0 { // It uses UUID so it should be 38
return snapshot, nil
}
}
}
return "", errors.New("no snapshot")
}
func RevertToSnapshot(vmName, snapshotID string) error {
return Prlctl("snapshot-switch", vmName, "--id", snapshotID)
}
func Start(vmName string) error {
return Prlctl("start", vmName)
}
func Status(vmName string) (StatusType, error) {
output, err := PrlctlOutput("list", vmName, "--no-header", "--output", "status")
if err != nil {
return NotFound, err
}
return StatusType(strings.TrimSpace(output)), nil
}
func WaitForStatus(vmName string, vmStatus StatusType, seconds int) error {
var status StatusType
var err error
for i := 0; i < seconds; i++ {
status, err = Status(vmName)
if err != nil {
return err
}
if status == vmStatus {
return nil
}
time.Sleep(time.Second)
}
return errors.New("VM " + vmName + " is in " + string(status) + " where it should be in " + string(vmStatus))
}
func TryExec(vmName string, seconds int, cmd ...string) error {
var err error
for i := 0; i < seconds; i++ {
_, err = Exec(vmName, cmd...)
if err == nil {
return nil
}
time.Sleep(time.Second)
}
return err
}
func Kill(vmName string) error {
return Prlctl("stop", vmName, "--kill")
}
func Delete(vmName string) error {
return Prlctl("delete", vmName)
}
func Unregister(vmName string) error {
return Prlctl("unregister", vmName)
}
func Mac(vmName string) (string, error) {
output, err := PrlctlOutput("list", "-i", vmName)
if err != nil {
return "", err
}
stdoutString := strings.TrimSpace(output)
re := regexp.MustCompile("net0.* mac=([0-9A-F]{12}) card=.*")
macMatch := re.FindAllStringSubmatch(stdoutString, 1)
if len(macMatch) != 1 {
return "", fmt.Errorf("MAC address for NIC: nic0 on Virtual Machine: %s not found", vmName)
}
mac := macMatch[0][1]
logrus.Debugf("Found MAC address for NIC: net0 - %s\n", mac)
return mac, nil
}
// IPAddress finds the IP address of a VM connected that uses DHCP by its MAC address
//
// Parses the file /Library/Preferences/Parallels/parallels_dhcp_leases
// file contain a list of DHCP leases given by Parallels Desktop
// Example line:
// 10.211.55.181="1418921112,1800,001c42f593fb,ff42f593fb000100011c25b9ff001c42f593fb"
// IP Address ="Lease expiry, Lease time, MAC, MAC or DUID"
func IPAddress(mac string) (string, error) {
if len(mac) != 12 {
return "", fmt.Errorf("not a valid MAC address: %s. It should be exactly 12 digits", mac)
}
leases, err := ioutil.ReadFile(dhcpLeases)
if err != nil {
return "", err
}
re := regexp.MustCompile("(.*)=\"(.*),(.*)," + strings.ToLower(mac) + ",.*\"")
mostRecentIP := ""
mostRecentLease := uint64(0)
for _, l := range re.FindAllStringSubmatch(string(leases), -1) {
ip := l[1]
expiry, _ := strconv.ParseUint(l[2], 10, 64)
leaseTime, _ := strconv.ParseUint(l[3], 10, 32)
logrus.Debugf("Found lease: %s for MAC: %s, expiring at %d, leased for %d s.\n", ip, mac, expiry, leaseTime)
if mostRecentLease <= expiry-leaseTime {
mostRecentIP = ip
mostRecentLease = expiry - leaseTime
}
}
if mostRecentIP == "" {
return "", fmt.Errorf("IP lease not found for MAC address %s in: %s", mac, dhcpLeases)
}
logrus.Debugf("Found IP lease: %s for MAC address %s\n", mostRecentIP, mac)
return mostRecentIP, nil
}
package path
import golang_path "path"
type unixPath struct{}
func (p *unixPath) Join(elem ...string) string {
return golang_path.Join(elem...)
}
func (p *unixPath) IsAbs(path string) bool {
path = golang_path.Clean(path)
return golang_path.IsAbs(path)
}
func (p *unixPath) IsRoot(path string) bool {
path = golang_path.Clean(path)
return golang_path.IsAbs(path) && golang_path.Dir(path) == path
}
func (p *unixPath) Contains(basePath, targetPath string) bool {
basePath = golang_path.Clean(basePath)
targetPath = golang_path.Clean(targetPath)
for {
if targetPath == basePath {
return true
}
if p.IsRoot(targetPath) || targetPath == "." {
return false
}
targetPath = golang_path.Dir(targetPath)
}
}
func NewUnixPath() Path {
return &unixPath{}
}
package process
import (
"io"
"os"
"os/exec"
"time"
)
type Commander interface {
Start() error
Wait() error
Process() *os.Process
}
type CommandOptions struct {
Dir string
Env []string
Stdout io.Writer
Stderr io.Writer
Stdin io.Reader
Logger Logger
GracefulKillTimeout time.Duration
ForceKillTimeout time.Duration
UseWindowsLegacyProcessStrategy bool
}
type osCmd struct {
internal *exec.Cmd
options CommandOptions
}
// NewOSCmd creates a new implementation of Commander using the os.Cmd from
// os/exec.
func NewOSCmd(executable string, args []string, options CommandOptions) Commander {
c := exec.Command(executable, args...)
c.Dir = options.Dir
c.Env = options.Env
c.Stdin = options.Stdin
c.Stdout = options.Stdout
c.Stderr = options.Stderr
return &osCmd{
internal: c,
options: options,
}
}
func (c *osCmd) Start() error {
setProcessGroup(c.internal, c.options.UseWindowsLegacyProcessStrategy)
return c.internal.Start()
}
func (c *osCmd) Wait() error {
return c.internal.Wait()
}
func (c *osCmd) Process() *os.Process {
return c.internal.Process
}
// +build darwin dragonfly freebsd linux netbsd openbsd
package process
import (
"os/exec"
"syscall"
)
func setProcessGroup(c *exec.Cmd, _ bool) {
c.SysProcAttr = &syscall.SysProcAttr{
Setpgid: true,
}
}
package process
import (
"errors"
"fmt"
"time"
"github.com/sirupsen/logrus"
)
// ErrProcessNotStarted is returned when we try to manipulated/interact with a
// process that hasn't started yet (still nil).
var ErrProcessNotStarted = errors.New("process not started yet")
// GracefulTimeout is the time a Killer should wait in general to the graceful
// termination to timeout.
const GracefulTimeout = 10 * time.Minute
// KillTimeout is the time a killer should wait in general for the kill command
// to finish.
const KillTimeout = 10 * time.Second
type killer interface {
Terminate()
ForceKill()
}
var newProcessKiller = newKiller
type KillWaiter interface {
KillAndWait(command Commander, waitCh chan error) error
}
type KillProcessError struct {
pid int
}
func (k *KillProcessError) Error() string {
return fmt.Sprintf("failed to kill process PID=%d, likely process is dormant", k.pid)
}
func (k *KillProcessError) Is(err error) bool {
_, ok := err.(*KillProcessError)
return ok
}
type osKillWait struct {
logger Logger
gracefulKillTimeout time.Duration
forceKillTimeout time.Duration
}
func NewOSKillWait(logger Logger, gracefulKillTimeout, forceKillTimeout time.Duration) KillWaiter {
return &osKillWait{
logger: logger,
gracefulKillTimeout: gracefulKillTimeout,
forceKillTimeout: forceKillTimeout,
}
}
// KillAndWait will take the specified process and terminate the process and
// wait util the waitCh returns or the graceful kill timer runs out after which
// a force kill on the process would be triggered.
func (kw *osKillWait) KillAndWait(command Commander, waitCh chan error) error {
process := command.Process()
if process == nil {
return ErrProcessNotStarted
}
log := kw.logger.WithFields(logrus.Fields{
"PID": process.Pid,
})
processKiller := newProcessKiller(log, command)
processKiller.Terminate()
select {
case err := <-waitCh:
return err
case <-time.After(kw.gracefulKillTimeout):
processKiller.ForceKill()
select {
case err := <-waitCh:
return err
case <-time.After(kw.forceKillTimeout):
return &KillProcessError{pid: process.Pid}
}
}
}
// +build darwin dragonfly freebsd linux netbsd openbsd
package process
import (
"syscall"
)
type unixKiller struct {
logger Logger
cmd Commander
}
func newKiller(logger Logger, cmd Commander) killer {
return &unixKiller{
logger: logger,
cmd: cmd,
}
}
func (pk *unixKiller) Terminate() {
if pk.cmd.Process() == nil {
return
}
err := syscall.Kill(pk.getPID(), syscall.SIGTERM)
if err != nil {
pk.logger.Warn("Failed to terminate process:", err)
// try to kill right-after
pk.ForceKill()
}
}
func (pk *unixKiller) ForceKill() {
if pk.cmd.Process() == nil {
return
}
err := syscall.Kill(pk.getPID(), syscall.SIGKILL)
if err != nil {
pk.logger.Warn("Failed to force-kill:", err)
}
}
// getPID will return the negative PID (-PID) which is the process group. The
// negative symbol comes from kill(2) https://linux.die.net/man/2/kill `If pid
// is less than -1, then sig is sent to every process in the process group whose
// ID is -pid.`
func (pk *unixKiller) getPID() int {
return pk.cmd.Process().Pid * -1
}
package prometheus
import (
"sync"
"github.com/prometheus/client_golang/prometheus"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
var numJobFailuresDesc = prometheus.NewDesc(
"gitlab_runner_failed_jobs_total",
"Total number of failed jobs",
[]string{"runner", "failure_reason"},
nil,
)
type failurePermutation struct {
runnerDescription string
reason common.JobFailureReason
}
type FailuresCollector struct {
lock sync.RWMutex
failures map[failurePermutation]int64
}
func (fc *FailuresCollector) RecordFailure(reason common.JobFailureReason, runnerDescription string) {
failure := failurePermutation{
runnerDescription: runnerDescription,
reason: reason,
}
fc.lock.Lock()
defer fc.lock.Unlock()
fc.failures[failure]++
}
func (fc *FailuresCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- numJobFailuresDesc
}
func (fc *FailuresCollector) Collect(ch chan<- prometheus.Metric) {
fc.lock.RLock()
defer fc.lock.RUnlock()
for failure, number := range fc.failures {
ch <- prometheus.MustNewConstMetric(
numJobFailuresDesc,
prometheus.CounterValue,
float64(number),
failure.runnerDescription,
string(failure.reason),
)
}
}
func NewFailuresCollector() *FailuresCollector {
return &FailuresCollector{
failures: make(map[failurePermutation]int64),
}
}
package prometheus
import (
"sync/atomic"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
)
var numErrorsDesc = prometheus.NewDesc(
"gitlab_runner_errors_total",
"The number of caught errors.",
[]string{"level"},
nil,
)
type LogHook struct {
errorsNumber map[logrus.Level]*int64
}
func (lh *LogHook) Levels() []logrus.Level {
return []logrus.Level{
logrus.PanicLevel,
logrus.FatalLevel,
logrus.ErrorLevel,
logrus.WarnLevel,
}
}
func (lh *LogHook) Fire(entry *logrus.Entry) error {
atomic.AddInt64(lh.errorsNumber[entry.Level], 1)
return nil
}
func (lh *LogHook) Describe(ch chan<- *prometheus.Desc) {
ch <- numErrorsDesc
}
func (lh *LogHook) Collect(ch chan<- prometheus.Metric) {
for _, level := range lh.Levels() {
number := float64(atomic.LoadInt64(lh.errorsNumber[level]))
ch <- prometheus.MustNewConstMetric(numErrorsDesc, prometheus.CounterValue, number, level.String())
}
}
func NewLogHook() LogHook {
lh := LogHook{}
levels := lh.Levels()
lh.errorsNumber = make(map[logrus.Level]*int64, len(levels))
for _, level := range levels {
lh.errorsNumber[level] = new(int64)
}
return lh
}
package helpers
import (
"crypto/rand"
"encoding/hex"
)
func GenerateRandomUUID(length int) (string, error) {
data := make([]byte, length)
_, err := rand.Read(data)
if err != nil {
return "", err
}
return hex.EncodeToString(data), nil
}
package retry
import (
"time"
"github.com/jpillora/backoff"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
const (
defaultRetryBackoffMin = 1 * time.Second
defaultRetryBackoffMax = 5 * time.Second
)
type Retryable interface {
Run() error
ShouldRetry(tries int, err error) bool
}
type Retry struct {
retryable Retryable
backoff *backoff.Backoff
}
func New(retry Retryable) *Retry {
return &Retry{
retryable: retry,
backoff: &backoff.Backoff{Min: defaultRetryBackoffMin, Max: defaultRetryBackoffMax},
}
}
func (r *Retry) Run() error {
var err error
var tries int
for {
tries++
err = r.retryable.Run()
if err == nil || !r.retryable.ShouldRetry(tries, err) {
break
}
time.Sleep(r.backoff.Duration())
}
return err
}
func WithLogrus(retry Retryable, log *logrus.Entry) Retryable {
return newRetryableDecorator(retry.Run, func(tries int, err error) bool {
shouldRetry := retry.ShouldRetry(tries, err)
if shouldRetry {
log.WithError(err).Warningln("Retrying...")
}
return shouldRetry
})
}
func WithBuildLog(retry Retryable, log *common.BuildLogger) Retryable {
return newRetryableDecorator(retry.Run, func(tries int, err error) bool {
shouldRetry := retry.ShouldRetry(tries, err)
if shouldRetry {
logger := log.WithFields(logrus.Fields{logrus.ErrorKey: err})
logger.Warningln("Retrying...")
}
return shouldRetry
})
}
type retryableDecorator struct {
run func() error
shouldRetry func(tries int, err error) bool
}
func newRetryableDecorator(run func() error, shouldRetry func(tries int, err error) bool) *retryableDecorator {
return &retryableDecorator{
run: run,
shouldRetry: shouldRetry,
}
}
func (d *retryableDecorator) Run() error {
return d.run()
}
func (d *retryableDecorator) ShouldRetry(tries int, err error) bool {
return d.shouldRetry(tries, err)
}
package secrets
import (
"fmt"
)
type ResolvingUnsupportedSecretError struct {
name string
}
func NewResolvingUnsupportedSecretError(name string) error {
return &ResolvingUnsupportedSecretError{name: name}
}
func (e *ResolvingUnsupportedSecretError) Error() string {
return fmt.Sprintf("trying to resolve unsupported secret: %s", e.name)
}
func (e *ResolvingUnsupportedSecretError) Is(err error) bool {
customErr, ok := err.(*ResolvingUnsupportedSecretError)
if !ok {
return false
}
return customErr.name == e.name
}
package vault
import (
"fmt"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/secrets"
"gitlab.com/gitlab-org/gitlab-runner/helpers/vault/service"
)
const (
resolverName = "vault"
)
var newVaultService = service.NewVault
type resolver struct {
secret common.Secret
}
func newResolver(secret common.Secret) common.SecretResolver {
return &resolver{
secret: secret,
}
}
func (v *resolver) Name() string {
return resolverName
}
func (v *resolver) IsSupported() bool {
return v.secret.Vault != nil
}
func (v *resolver) Resolve() (string, error) {
if !v.IsSupported() {
return "", secrets.NewResolvingUnsupportedSecretError(resolverName)
}
secret := v.secret.Vault
url := secret.Server.URL
s, err := newVaultService(url, secret)
if err != nil {
return "", err
}
data, err := s.GetField(secret, secret)
if err != nil {
return "", err
}
return fmt.Sprintf("%v", data), nil
}
func init() {
common.GetSecretResolverRegistry().Register(newResolver)
}
package sentry
import (
"errors"
"fmt"
"os"
"runtime"
"github.com/getsentry/raven-go"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
type LogHook struct {
client *raven.Client
}
func (s *LogHook) Levels() []logrus.Level {
return []logrus.Level{
logrus.PanicLevel,
logrus.FatalLevel,
logrus.ErrorLevel,
}
}
func (s *LogHook) Fire(entry *logrus.Entry) error {
if s.client == nil {
return nil
}
tags := make(map[string]string)
for key, value := range entry.Data {
tags[key] = fmt.Sprint(value)
}
switch entry.Level {
case logrus.PanicLevel:
s.client.CaptureErrorAndWait(errors.New(entry.Message), tags)
case logrus.FatalLevel:
s.client.CaptureErrorAndWait(errors.New(entry.Message), tags)
case logrus.ErrorLevel:
s.client.CaptureError(errors.New(entry.Message), tags)
}
return nil
}
func NewLogHook(dsn string) (lh LogHook, err error) {
tags := make(map[string]string)
tags["built"] = common.BUILT
tags["version"] = common.VERSION
tags["revision"] = common.REVISION
tags["branch"] = common.BRANCH
tags["go-version"] = runtime.Version()
tags["go-os"] = runtime.GOOS
tags["go-arch"] = runtime.GOARCH
tags["hostname"], _ = os.Hostname()
client, err := raven.NewWithTags(dsn, tags)
if err != nil {
return
}
lh.client = client
return
}
package service_helpers
import "os"
func SysvScript() string {
switch {
case isDebianSysv():
return sysvDebianScript
case isRedhatSysv():
return sysvRedhatScript
}
return ""
}
func isDebianSysv() bool {
if _, err := os.Stat("/lib/lsb/init-functions"); err != nil {
return false
}
if _, err := os.Stat("/sbin/start-stop-daemon"); err != nil {
return false
}
return true
}
func isRedhatSysv() bool {
if _, err := os.Stat("/etc/rc.d/init.d/functions"); err != nil {
return false
}
return true
}
const sysvDebianScript = `#! /bin/bash
### BEGIN INIT INFO
# Provides: {{.Path}}
# Required-Start: $local_fs $remote_fs $network $syslog
# Required-Stop: $local_fs $remote_fs $network $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: {{.DisplayName}}
# Description: {{.Description}}
### END INIT INFO
DESC="{{.Description}}"
USER="{{.UserName}}"
NAME="{{.Name}}"
PIDFILE="/var/run/$NAME.pid"
# Read configuration variable file if it is present
[ -r /etc/default/$NAME ] && . /etc/default/$NAME
# Define LSB log_* functions.
. /lib/lsb/init-functions
## Check to see if we are running as root first.
if [ "$(id -u)" != "0" ]; then
echo "This script must be run as root"
exit 1
fi
do_start() {
start-stop-daemon --start \
{{if .ChRoot}}--chroot {{.ChRoot|cmd}}{{end}} \
{{if .WorkingDirectory}}--chdir {{.WorkingDirectory|cmd}}{{end}} \
{{if .UserName}} --chuid {{.UserName|cmd}}{{end}} \
--pidfile "$PIDFILE" \
--background \
--make-pidfile \
--exec {{.Path}} -- {{range .Arguments}} {{.|cmd}}{{end}}
}
do_stop() {
start-stop-daemon --stop \
{{if .UserName}} --chuid {{.UserName|cmd}}{{end}} \
--pidfile "$PIDFILE" \
--quiet
}
case "$1" in
start)
log_daemon_msg "Starting $DESC"
do_start
log_end_msg $?
;;
stop)
log_daemon_msg "Stopping $DESC"
do_stop
log_end_msg $?
;;
restart)
$0 stop
$0 start
;;
status)
status_of_proc -p "$PIDFILE" "$DAEMON" "$DESC"
;;
*)
echo "Usage: sudo service $0 {start|stop|restart|status}" >&2
exit 1
;;
esac
exit 0
`
const sysvRedhatScript = `#!/bin/sh
# For RedHat and cousins:
# chkconfig: - 99 01
# description: {{.Description}}
# processname: {{.Path}}
# Source function library.
. /etc/rc.d/init.d/functions
name="{{.Name}}"
desc="{{.Description}}"
user="{{.UserName}}"
cmd={{.Path}}
args="{{range .Arguments}} {{.|cmd}}{{end}}"
lockfile=/var/lock/subsys/$name
pidfile=/var/run/$name.pid
# Source networking configuration.
[ -r /etc/sysconfig/$name ] && . /etc/sysconfig/$name
start() {
echo -n $"Starting $desc: "
daemon \
{{if .UserName}}--user=$user{{end}} \
{{if .WorkingDirectory}}--chdir={{.WorkingDirectory|cmd}}{{end}} \
"$cmd $args </dev/null >/dev/null 2>/dev/null & echo \$! > $pidfile"
retval=$?
[ $retval -eq 0 ] && touch $lockfile
echo
return $retval
}
stop() {
echo -n $"Stopping $desc: "
killproc -p $pidfile $cmd -TERM
retval=$?
[ $retval -eq 0 ] && rm -f $lockfile
rm -f $pidfile
echo
return $retval
}
restart() {
stop
start
}
reload() {
echo -n $"Reloading $desc: "
killproc -p $pidfile $cmd -HUP
RETVAL=$?
echo
}
force_reload() {
restart
}
rh_status() {
status -p $pidfile $cmd
}
rh_status_q() {
rh_status >/dev/null 2>&1
}
case "$1" in
start)
rh_status_q && exit 0
$1
;;
stop)
rh_status_q || exit 0
$1
;;
restart)
$1
;;
reload)
rh_status_q || exit 7
$1
;;
force-reload)
force_reload
;;
status)
rh_status
;;
condrestart|try-restart)
rh_status_q || exit 0
;;
*)
echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
exit 2
esac
`
package service_helpers
import (
"github.com/kardianos/service"
"github.com/sirupsen/logrus"
)
func New(i service.Interface, c *service.Config) (service.Service, error) {
s, err := service.New(i, c)
if err == service.ErrNoServiceSystemDetected {
logrus.Warningln("No service system detected. Some features may not work!")
return &SimpleService{
i: i,
c: c,
}, nil
}
return s, err
}
package service_helpers
import (
"errors"
"os"
"os/signal"
"syscall"
"github.com/kardianos/service"
)
var (
// ErrNotSupported is returned when specific feature is not supported.
ErrNotSupported = errors.New("not supported")
)
//nolint:deadcode
type stopStarter interface {
Start(service.Service) error
Stop(service.Service) error
}
type SimpleService struct {
i service.Interface
c *service.Config
}
// Run should be called shortly after the program entry point.
// After Interface.Stop has finished running, Run will stop blocking.
// After Run stops blocking, the program must exit shortly after.
func (s *SimpleService) Run() (err error) {
err = s.i.Start(s)
if err != nil {
return err
}
sigChan := make(chan os.Signal, 3)
signal.Notify(sigChan, syscall.SIGTERM, os.Interrupt)
<-sigChan
return s.i.Stop(s)
}
// Start signals to the OS service manager the given service should start.
func (s *SimpleService) Start() error {
return service.ErrNoServiceSystemDetected
}
// Stop signals to the OS service manager the given service should stop.
func (s *SimpleService) Stop() error {
return ErrNotSupported
}
// Restart signals to the OS service manager the given service should stop then start.
func (s *SimpleService) Restart() error {
return ErrNotSupported
}
// Install setups up the given service in the OS service manager. This may require
// greater rights. Will return an error if it is already installed.
func (s *SimpleService) Install() error {
return ErrNotSupported
}
// Uninstall removes the given service from the OS service manager. This may require
// greater rights. Will return an error if the service is not present.
func (s *SimpleService) Uninstall() error {
return ErrNotSupported
}
// Status returns nil if the given service is running.
// Will return an error if the service is not running or is not present.
func (s *SimpleService) Status() (service.Status, error) {
return service.StatusUnknown, ErrNotSupported
}
// Logger opens and returns a system logger. If the user program is running
// interactively rather then as a service, the returned logger will write to
// os.Stderr. If errs is non-nil errors will be sent on errs as well as
// returned from Logger's functions.
func (s *SimpleService) Logger(errs chan<- error) (service.Logger, error) {
return service.ConsoleLogger, nil
}
// SystemLogger opens and returns a system logger. If errs is non-nil errors
// will be sent on errs as well as returned from Logger's functions.
func (s *SimpleService) SystemLogger(errs chan<- error) (service.Logger, error) {
return nil, ErrNotSupported
}
// String displays the name of the service. The display name if present,
// otherwise the name.
func (s *SimpleService) String() string {
return "SimpleService"
}
// Platform displays the name of the system that manages the service.
// In most cases this will be the same as service.Platform().
func (s *SimpleService) Platform() string {
return service.Platform()
}
package helpers
// https://github.com/zimbatm/direnv/blob/master/shell.go
import (
"bytes"
"encoding/hex"
"strings"
)
/*
* Escaping
*/
const (
ACK = 6
TAB = 9
LF = 10
CR = 13
US = 31
SPACE = 32
AMPERSTAND = 38
SINGLE_QUOTE = 39
PLUS = 43
NINE = 57
QUESTION = 63
LOWERCASE_Z = 90
OPEN_BRACKET = 91
BACKSLASH = 92
UNDERSCORE = 95
CLOSE_BRACKET = 93
BACKTICK = 96
TILDA = 126
DEL = 127
)
type shellEscaper struct {
}
//nolint:lll
// ShellEscape is taken from
// https://github.com/solidsnack/shell-escape/blob/056c7b308be32ffeafec815907699f6c27536b1e/Data/ByteString/ShellEscape/Bash.hs
/*
A Bash escaped string. The strings are wrapped in @$\'...\'@ if any
bytes within them must be escaped; otherwise, they are left as is.
Newlines and other control characters are represented as ANSI escape
sequences. High bytes are represented as hex codes. Thus Bash escaped
strings will always fit on one line and never contain non-ASCII bytes.
*/
func ShellEscape(str string) string {
e := newShellEscaper()
outStr := e.getEscapedString(str)
return outStr
}
func newShellEscaper() *shellEscaper {
e := &shellEscaper{}
return e
}
func (e *shellEscaper) hex(char byte, out *bytes.Buffer) bool {
data := []byte{BACKSLASH, 'x', 0, 0}
hex.Encode(data[2:], []byte{char})
out.Write(data)
return true
}
func (e *shellEscaper) backslash(char byte, out *bytes.Buffer) bool {
out.Write([]byte{BACKSLASH, char})
return true
}
func (e *shellEscaper) escaped(str string, out *bytes.Buffer) bool {
out.WriteString(str)
return true
}
func (e *shellEscaper) quoted(char byte, out *bytes.Buffer) bool {
out.WriteByte(char)
return true
}
func (e *shellEscaper) literal(char byte, out *bytes.Buffer) bool {
out.WriteByte(char)
return false
}
func (e *shellEscaper) getEscapedString(str string) string {
if str == "" {
return "''"
}
escape := false
in := []byte(str)
out := bytes.NewBuffer(make([]byte, 0, len(str)*2))
for _, c := range in {
if e.processChar(c, out) {
escape = true
}
}
outStr := out.String()
if escape {
outStr = "$'" + outStr + "'"
}
return outStr
}
func (e *shellEscaper) processChar(char byte, out *bytes.Buffer) bool {
switch {
case char == TAB:
return e.escaped(`\t`, out)
case char == LF:
return e.escaped(`\n`, out)
case char == CR:
return e.escaped(`\r`, out)
case char <= US:
return e.hex(char, out)
case char <= AMPERSTAND:
return e.quoted(char, out)
case char == SINGLE_QUOTE:
return e.backslash(char, out)
case char <= PLUS:
return e.quoted(char, out)
case char <= NINE:
return e.literal(char, out)
case char <= QUESTION:
return e.quoted(char, out)
case char <= LOWERCASE_Z:
return e.literal(char, out)
case char == OPEN_BRACKET:
return e.quoted(char, out)
case char == BACKSLASH:
return e.backslash(char, out)
case char <= CLOSE_BRACKET:
return e.quoted(char, out)
case char == UNDERSCORE:
return e.literal(char, out)
case char <= BACKTICK:
return e.quoted(char, out)
case char <= TILDA:
return e.quoted(char, out)
case char == DEL:
return e.hex(char, out)
default:
return e.hex(char, out)
}
}
func ToBackslash(path string) string {
return strings.ReplaceAll(path, "/", "\\")
}
func ToSlash(path string) string {
return strings.ReplaceAll(path, "\\", "/")
}
package helpers
func ShortenToken(token string) string {
if len(token) >= 8 {
return token[0:8]
}
return token
}
package ssh
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"strings"
"time"
"golang.org/x/crypto/ssh"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
)
type Client struct {
Config
Stdout io.Writer
Stderr io.Writer
ConnectRetries int
client *ssh.Client
}
type Command struct {
Environment []string
Command []string
Stdin string
}
type ExitError struct {
Inner error
}
func (e *ExitError) Error() string {
if e.Inner == nil {
return "error"
}
return e.Inner.Error()
}
func (e *ExitError) ExitCode() int {
var cryptoExitError *ssh.ExitError
if errors.As(e.Inner, &cryptoExitError) {
return cryptoExitError.ExitStatus()
}
return 0
}
func (s *Client) getSSHKey(identityFile string) (key ssh.Signer, err error) {
buf, err := ioutil.ReadFile(identityFile)
if err != nil {
return nil, err
}
key, err = ssh.ParsePrivateKey(buf)
return key, err
}
func (s *Client) getSSHAuthMethods() ([]ssh.AuthMethod, error) {
var methods []ssh.AuthMethod
methods = append(methods, ssh.Password(s.Password))
if s.IdentityFile != "" {
key, err := s.getSSHKey(s.IdentityFile)
if err != nil {
return nil, err
}
methods = append(methods, ssh.PublicKeys(key))
}
return methods, nil
}
func (s *Client) Connect() error {
if s.Host == "" {
s.Host = "localhost"
}
if s.User == "" {
s.User = "root"
}
if s.Port == "" {
s.Port = "22"
}
methods, err := s.getSSHAuthMethods()
if err != nil {
return fmt.Errorf("getSSHAuthMethods error: %w", err)
}
config := &ssh.ClientConfig{
User: s.User,
Auth: methods,
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
}
connectRetries := s.ConnectRetries
if connectRetries == 0 {
connectRetries = 3
}
var finalError error
for i := 0; i < connectRetries; i++ {
client, err := ssh.Dial("tcp", s.Host+":"+s.Port, config)
if err == nil {
s.client = client
return nil
}
time.Sleep(sshRetryInterval * time.Second)
finalError = fmt.Errorf("ssh Dial() error: %w", err)
}
return finalError
}
func (s *Client) Exec(cmd string) error {
if s.client == nil {
return errors.New("not connected")
}
session, err := s.client.NewSession()
if err != nil {
return err
}
session.Stdout = s.Stdout
session.Stderr = s.Stderr
err = session.Run(cmd)
_ = session.Close()
return err
}
func (s *Command) fullCommand() string {
var arguments []string
for _, part := range s.Command {
arguments = append(arguments, helpers.ShellEscape(part))
}
return strings.Join(arguments, " ")
}
func (s *Client) Run(ctx context.Context, cmd Command) error {
if s.client == nil {
return errors.New("not connected")
}
session, err := s.client.NewSession()
if err != nil {
return err
}
defer func() { _ = session.Close() }()
var envVariables bytes.Buffer
for _, keyValue := range cmd.Environment {
envVariables.WriteString("export " + helpers.ShellEscape(keyValue) + "\n")
}
session.Stdin = io.MultiReader(
&envVariables,
bytes.NewBufferString(cmd.Stdin),
)
session.Stdout = s.Stdout
session.Stderr = s.Stderr
err = session.Start(cmd.fullCommand())
if err != nil {
return err
}
waitCh := make(chan error)
go func() {
err := session.Wait()
if _, ok := err.(*ssh.ExitError); ok {
err = &ExitError{Inner: err}
}
waitCh <- err
}()
select {
case <-ctx.Done():
_ = session.Signal(ssh.SIGKILL)
_ = session.Close()
return <-waitCh
case err := <-waitCh:
return err
}
}
func (s *Client) Cleanup() {
if s.client != nil {
_ = s.client.Close()
}
}
package test
import (
"os"
"runtime"
"testing"
)
const (
OSWindows = "windows"
OSLinux = "linux"
)
func SkipIfGitLabCI(t *testing.T) {
_, ok := os.LookupEnv("CI")
if ok {
t.Skipf("Skipping test on CI builds: %s", t.Name())
}
}
func SkipIfGitLabCIOn(t *testing.T, os string) {
if runtime.GOOS != os {
return
}
SkipIfGitLabCI(t)
}
func SkipIfGitLabCIWithMessage(t *testing.T, msg string) {
_, ok := os.LookupEnv("CI")
if ok {
t.Skipf("Skipping test on CI builds: %s - %s", t.Name(), msg)
}
}
package timeperiod
import (
"time"
"github.com/gorhill/cronexpr"
)
type TimePeriod struct {
expressions []*cronexpr.Expression
location *time.Location
GetCurrentTime func() time.Time
}
func (t *TimePeriod) InPeriod() bool {
now := t.GetCurrentTime().In(t.location)
for _, expression := range t.expressions {
nextIn := expression.Next(now)
timeSince := now.Sub(nextIn)
if -time.Second <= timeSince && timeSince <= time.Second {
return true
}
}
return false
}
func TimePeriods(periods []string, timezone string) (*TimePeriod, error) {
return TimePeriodsWithTimer(periods, timezone, time.Now)
}
func TimePeriodsWithTimer(periods []string, timezone string, timer func() time.Time) (*TimePeriod, error) {
var expressions []*cronexpr.Expression
for _, period := range periods {
expression, err := cronexpr.Parse(period)
if err != nil {
return nil, err
}
expressions = append(expressions, expression)
}
// if not set, default to system setting (the empty string would mean UTC)
if timezone == "" {
timezone = "Local"
}
location, err := time.LoadLocation(timezone)
if err != nil {
return nil, err
}
timePeriod := &TimePeriod{
expressions: expressions,
location: location,
GetCurrentTime: timer,
}
return timePeriod, nil
}
package ca_chain
import (
"bytes"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/pem"
"fmt"
"io"
"strings"
"github.com/sirupsen/logrus"
)
const (
pemTypeCertificate = "CERTIFICATE"
)
type pemEncoder func(out io.Writer, b *pem.Block) error
type Builder interface {
fmt.Stringer
BuildChainFromTLSConnectionState(TLS *tls.ConnectionState) error
}
func NewBuilder(logger logrus.FieldLogger) Builder {
logger = logger.
WithField("context", "certificate-chain-build")
return &defaultBuilder{
certificates: make([]*x509.Certificate, 0),
seenCertificates: make(map[string]bool),
resolver: newChainResolver(
newURLResolver(logger),
newVerifyResolver(logger),
),
encodePEM: pem.Encode,
logger: logger,
}
}
type defaultBuilder struct {
certificates []*x509.Certificate
seenCertificates map[string]bool
resolver resolver
encodePEM pemEncoder
logger logrus.FieldLogger
}
func (b *defaultBuilder) BuildChainFromTLSConnectionState(tls *tls.ConnectionState) error {
for _, verifiedChain := range tls.VerifiedChains {
b.logger.
WithField("chain-leaf", fmt.Sprintf("%v", verifiedChain)).
Debug("Processing chain")
err := b.fetchCertificatesFromVerifiedChain(verifiedChain)
if err != nil {
return fmt.Errorf("error while fetching certificates into the CA Chain: %w", err)
}
}
return nil
}
func (b *defaultBuilder) fetchCertificatesFromVerifiedChain(verifiedChain []*x509.Certificate) error {
var err error
if len(verifiedChain) < 1 {
return nil
}
verifiedChain, err = b.resolver.Resolve(verifiedChain)
if err != nil {
return fmt.Errorf("couldn't resolve certificates chain from the leaf certificate: %w", err)
}
for _, certificate := range verifiedChain {
b.addCertificate(certificate)
}
return nil
}
func (b *defaultBuilder) addCertificate(certificate *x509.Certificate) {
signature := hex.EncodeToString(certificate.Signature)
if b.seenCertificates[signature] {
return
}
b.seenCertificates[signature] = true
b.certificates = append(b.certificates, certificate)
}
func (b *defaultBuilder) String() string {
out := bytes.NewBuffer(nil)
for _, certificate := range b.certificates {
err := b.encodePEM(out, &pem.Block{Type: pemTypeCertificate, Bytes: certificate.Raw})
if err != nil {
b.logger.
WithError(err).
Warning("Failed to encode certificate from chain")
}
}
return strings.TrimSpace(out.String())
}
// Inspired by https://github.com/zakjan/cert-chain-resolver/blob/1.0.3/certUtil/io.go
// which is licensed on a MIT license.
//
// Shout out to Jan Žák (http://zakjan.cz) original author of `certUtil` package and other
// contributors who updated it!
package ca_chain
import (
"bytes"
"crypto/x509"
"encoding/pem"
"fmt"
"strings"
"github.com/fullsailor/pkcs7"
"github.com/sirupsen/logrus"
)
const (
pemStart = "-----BEGIN "
pemCertBlockType = "CERTIFICATE"
)
type ErrorInvalidCertificate struct {
inner error
nonCertBlockType bool
nilBlock bool
}
func (e *ErrorInvalidCertificate) Error() string {
msg := []string{"invalid certificate"}
switch {
case e.nilBlock:
msg = append(msg, "empty PEM block")
case e.nonCertBlockType:
msg = append(msg, "non-certificate PEM block")
case e.inner != nil:
msg = append(msg, e.inner.Error())
}
return strings.Join(msg, ": ")
}
func decodeCertificate(data []byte) (*x509.Certificate, error) {
if isPEM(data) {
block, _ := pem.Decode(data)
if block == nil {
return nil, &ErrorInvalidCertificate{nilBlock: true}
}
if block.Type != pemCertBlockType {
return nil, &ErrorInvalidCertificate{nonCertBlockType: true}
}
data = block.Bytes
}
cert, err := x509.ParseCertificate(data)
if err == nil {
return cert, nil
}
p, err := pkcs7.Parse(data)
if err == nil {
// pkcs7.Parse() can return a nil payload if no certs were decoded
if p == nil || len(p.Certificates) == 0 {
return nil, nil
}
return p.Certificates[0], nil
}
return nil, &ErrorInvalidCertificate{inner: err}
}
func isPEM(data []byte) bool {
return bytes.HasPrefix(data, []byte(pemStart))
}
func isSelfSigned(cert *x509.Certificate) bool {
return cert.CheckSignatureFrom(cert) == nil
}
func prepareCertificateLogger(logger logrus.FieldLogger, cert *x509.Certificate) logrus.FieldLogger {
return preparePrefixedCertificateLogger(logger, cert, "")
}
func preparePrefixedCertificateLogger(
logger logrus.FieldLogger,
cert *x509.Certificate,
prefix string,
) logrus.FieldLogger {
return logger.
WithFields(logrus.Fields{
fmt.Sprintf("%sSubject", prefix): cert.Subject.CommonName,
fmt.Sprintf("%sIssuer", prefix): cert.Issuer.CommonName,
fmt.Sprintf("%sSerial", prefix): cert.SerialNumber.String(),
fmt.Sprintf("%sIssuerCertURL", prefix): cert.IssuingCertificateURL,
})
}
func verifyCertificate(cert *x509.Certificate) ([][]*x509.Certificate, error) {
return cert.Verify(x509.VerifyOptions{})
}
// Inspired by https://github.com/zakjan/cert-chain-resolver/blob/1.0.3/certUtil/chain.go
// which is licensed on a MIT license.
//
// Shout out to Jan Žák (http://zakjan.cz) original author of `certUtil` package and other
// contributors who updated it!
package ca_chain
import (
"crypto/x509"
"fmt"
)
type chainResolver struct {
urlResolver resolver
verifyResolver resolver
}
func newChainResolver(urlResolver, verifyResolver resolver) resolver {
return &chainResolver{
urlResolver: urlResolver,
verifyResolver: verifyResolver,
}
}
func (r *chainResolver) Resolve(certs []*x509.Certificate) ([]*x509.Certificate, error) {
certs, err := r.urlResolver.Resolve(certs)
if err != nil {
return nil, fmt.Errorf("error while resolving certificates chain with URL: %w", err)
}
certs, err = r.verifyResolver.Resolve(certs)
if err != nil {
return nil, fmt.Errorf("error while resolving certificates chain with verification: %w", err)
}
return certs, err
}
// Inspired by https://github.com/zakjan/cert-chain-resolver/blob/1.0.3/certUtil/chain.go
// which is licensed on a MIT license.
//
// Shout out to Jan Žák (http://zakjan.cz) original author of `certUtil` package and other
// contributors who updated it!
package ca_chain
import (
"crypto/x509"
"fmt"
"io/ioutil"
"net/http"
"time"
"github.com/sirupsen/logrus"
)
const defaultURLResolverLoopLimit = 15
const defaultURLResolverFetchTimeout = 15 * time.Second
type fetcher interface {
Fetch(url string) ([]byte, error)
}
type httpFetcher struct {
client *http.Client
}
func newHTTPFetcher(timeout time.Duration) *httpFetcher {
return &httpFetcher{
client: &http.Client{
Timeout: timeout,
},
}
}
func (f *httpFetcher) Fetch(url string) ([]byte, error) {
resp, err := f.client.Get(url)
if resp != nil {
defer func() { _ = resp.Body.Close() }()
}
if err != nil {
return nil, err
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return data, nil
}
type decoder func(data []byte) (*x509.Certificate, error)
type urlResolver struct {
logger logrus.FieldLogger
fetcher fetcher
decoder decoder
loopLimit int
}
func newURLResolver(logger logrus.FieldLogger) resolver {
return &urlResolver{
logger: logger,
fetcher: newHTTPFetcher(defaultURLResolverFetchTimeout),
decoder: decodeCertificate,
loopLimit: defaultURLResolverLoopLimit,
}
}
func (r *urlResolver) Resolve(certs []*x509.Certificate) ([]*x509.Certificate, error) {
if len(certs) < 1 {
return nil, nil
}
loop := 0
for {
loop++
if loop >= r.loopLimit {
r.
logger.
Warning("urlResolver loop limit exceeded; exiting the loop")
break
}
certificate := certs[len(certs)-1]
log := prepareCertificateLogger(r.logger, certificate)
if certificate.IssuingCertificateURL == nil {
log.Debug("Certificate doesn't provide parent URL: exiting the loop")
break
}
newCert, err := r.fetchIssuerCertificate(certificate)
if err != nil {
return nil, fmt.Errorf("error while fetching issuer certificate: %w", err)
}
if newCert == nil {
log.Debug("Fetched issuer certificate file does not contain any certificates: exiting the loop")
break
}
certs = append(certs, newCert)
if isSelfSigned(newCert) {
log.Debug("Fetched issuer certificate is a ROOT certificate so exiting the loop")
break
}
}
return certs, nil
}
func (r *urlResolver) fetchIssuerCertificate(cert *x509.Certificate) (*x509.Certificate, error) {
log := prepareCertificateLogger(r.logger, cert).
WithField("method", "fetchIssuerCertificate")
issuerURL := cert.IssuingCertificateURL[0]
data, err := r.fetcher.Fetch(issuerURL)
if err != nil {
log.
WithError(err).
WithField("issuerURL", issuerURL).
Warning("Remote certificate fetching error")
return nil, fmt.Errorf("remote fetch failure: %w", err)
}
newCert, err := r.decoder(data)
if err != nil {
log.
WithError(err).
Warning("Certificate decoding error")
return nil, fmt.Errorf("decoding failure: %w", err)
}
if newCert == nil {
log.Debug("Issuer certificate file decoded properly but did not include any certificates")
return nil, nil
}
preparePrefixedCertificateLogger(log, newCert, "newCert").
Debug("Appending the certificate to the chain")
return newCert, nil
}
// Inspired by https://github.com/zakjan/cert-chain-resolver/blob/1.0.3/certUtil/chain.go
// which is licensed on a MIT license.
//
// Shout out to Jan Žák (http://zakjan.cz) original author of `certUtil` package and other
// contributors who updated it!
package ca_chain
import (
"crypto/x509"
"fmt"
"github.com/sirupsen/logrus"
)
type verifier func(cert *x509.Certificate) ([][]*x509.Certificate, error)
type verifyResolver struct {
logger logrus.FieldLogger
verifier verifier
}
func newVerifyResolver(logger logrus.FieldLogger) resolver {
return &verifyResolver{
logger: logger,
verifier: verifyCertificate,
}
}
func (r *verifyResolver) Resolve(certs []*x509.Certificate) ([]*x509.Certificate, error) {
if len(certs) < 1 {
return certs, nil
}
lastCert := certs[len(certs)-1]
if isSelfSigned(lastCert) {
return certs, nil
}
prepareCertificateLogger(r.logger, lastCert).
Debug("Verifying last certificate to find the final root certificate")
verifyChains, err := r.verifier(lastCert)
if err != nil {
_, ok := err.(x509.UnknownAuthorityError)
if ok {
prepareCertificateLogger(r.logger, lastCert).
WithError(err).
Warning("Last certificate signed by unknown authority; will not update the chain")
return certs, nil
}
return nil, fmt.Errorf("error while verifying last certificate from the chain: %w", err)
}
for _, cert := range verifyChains[0] {
if lastCert.Equal(cert) {
continue
}
prepareCertificateLogger(r.logger, cert).
Debug("Adding cert from verify chain to the final chain")
certs = append(certs, cert)
}
return certs, nil
}
package trace
import (
"errors"
"fmt"
"hash"
"hash/crc32"
"io"
"io/ioutil"
"os"
"sort"
"sync"
"unicode/utf8"
"golang.org/x/text/encoding"
"golang.org/x/text/transform"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
)
const defaultBytesLimit = 4 * 1024 * 1024 // 4MB
var errLogLimitExceeded = errors.New("log limit exceeded")
type Buffer struct {
lock sync.RWMutex
lw *limitWriter
w io.WriteCloser
logFile *os.File
checksum hash.Hash32
}
type inverseLengthSort []string
func (s inverseLengthSort) Len() int {
return len(s)
}
func (s inverseLengthSort) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}
func (s inverseLengthSort) Less(i, j int) bool {
return len(s[i]) > len(s[j])
}
func (b *Buffer) SetMasked(values []string) {
b.lock.Lock()
defer b.lock.Unlock()
// close existing writer to flush data
if b.w != nil {
b.w.Close()
}
defaultTransformers := []transform.Transformer{
encoding.Replacement.NewEncoder(),
}
transformers := make([]transform.Transformer, 0, len(values)+len(defaultTransformers))
sort.Sort(inverseLengthSort(values))
seen := make(map[string]struct{})
for _, value := range values {
if _, ok := seen[value]; ok {
continue
}
seen[value] = struct{}{}
transformers = append(transformers, newPhraseTransform(value))
}
transformers = append(transformers, defaultTransformers...)
b.w = transform.NewWriter(b.lw, transform.Chain(transformers...))
}
func (b *Buffer) SetLimit(size int) {
b.lock.Lock()
defer b.lock.Unlock()
b.lw.limit = int64(size)
}
func (b *Buffer) Size() int {
b.lock.RLock()
defer b.lock.RUnlock()
if b.lw == nil {
return 0
}
return int(b.lw.written)
}
func (b *Buffer) Bytes(offset, n int) ([]byte, error) {
return ioutil.ReadAll(io.NewSectionReader(b.logFile, int64(offset), int64(n)))
}
func (b *Buffer) Write(p []byte) (int, error) {
b.lock.Lock()
defer b.lock.Unlock()
src := p
var n int
for len(src) > 0 {
written, err := b.w.Write(src)
// if we get a log limit exceeded error, we've written the log limit
// notice out to the log and will now silently not write any additional
// data: we return len(p), nil so the caller continues as normal.
if err == errLogLimitExceeded {
return len(p), nil
}
if err != nil {
return n, err
}
// the text/transformer implementation can return n < len(p) without an
// error. For this reason, we continue writing whatever data is left
// unless nothing was written (therefore zero progress) on our call to
// Write().
if written == 0 {
return n, io.ErrShortWrite
}
src = src[written:]
n += written
}
return n, nil
}
func (b *Buffer) Finish() {
b.lock.RLock()
defer b.lock.RUnlock()
if b.w != nil {
_ = b.w.Close()
}
}
func (b *Buffer) Close() {
_ = b.logFile.Close()
_ = os.Remove(b.logFile.Name())
}
func (b *Buffer) Checksum() string {
b.lock.RLock()
defer b.lock.RUnlock()
return fmt.Sprintf("crc32:%08x", b.checksum.Sum32())
}
type limitWriter struct {
w io.Writer
written int64
limit int64
}
func (w *limitWriter) Write(p []byte) (int, error) {
capacity := w.limit - w.written
if capacity <= 0 {
return 0, errLogLimitExceeded
}
if int64(len(p)) >= capacity {
p = truncateSafeUTF8(p, capacity)
n, err := w.w.Write(p)
if err == nil {
err = errLogLimitExceeded
}
if n < 0 {
n = 0
}
w.written += int64(n)
w.writeLimitExceededMessage()
return n, err
}
n, err := w.w.Write(p)
if n < 0 {
n = 0
}
w.written += int64(n)
return n, err
}
func (w *limitWriter) writeLimitExceededMessage() {
n, _ := fmt.Fprintf(
w.w,
"\n%sJob's log exceeded limit of %v bytes.\n"+
"Job execution will continue but no more output will be collected.%s\n",
helpers.ANSI_BOLD_YELLOW,
w.limit,
helpers.ANSI_RESET,
)
w.written += int64(n)
}
func New() (*Buffer, error) {
logFile, err := ioutil.TempFile("", "trace")
if err != nil {
return nil, err
}
buffer := &Buffer{
logFile: logFile,
checksum: crc32.NewIEEE(),
}
buffer.lw = &limitWriter{
w: io.MultiWriter(buffer.logFile, buffer.checksum),
written: 0,
limit: defaultBytesLimit,
}
buffer.SetMasked(nil)
return buffer, nil
}
// truncateSafeUTF8 truncates a job log at the capacity but avoids
// breaking up a multi-byte UTF-8 character.
func truncateSafeUTF8(p []byte, capacity int64) []byte {
for i := 0; i < 4; i++ {
r, s := utf8.DecodeLastRune(p[:capacity])
if r == utf8.RuneError && s == 1 {
capacity--
continue
}
break
}
return p[:capacity]
}
package trace
import (
"bytes"
"golang.org/x/text/transform"
)
const (
// mask is the string that replaces any found sensitive information
mask = "[MASKED]"
// safeTokens are tokens that cannot appear in secret phrases
// and allows calling writers to set a safe boundary at which data written
// isn't buffered before being flushed to the underlying writer.
safeTokens = "\r\n"
)
// newPhraseTransform returns a transform.Transformer that replaces the `phrase`
// with [MASKED]
func newPhraseTransform(phrase string) transform.Transformer {
return phraseTransform(phrase)
}
type phraseTransform []byte
func (phraseTransform) Reset() {}
func (t phraseTransform) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
for {
// copy up until phrase
i := bytes.Index(src[nSrc:], t)
if i == -1 {
break
}
err = copyn(dst, src, &nDst, &nSrc, i)
if err != nil {
return nDst, nSrc, err
}
// replace phrase
err = replace(dst, &nDst, &nSrc, []byte(mask), len(t))
if err != nil {
return nDst, nSrc, err
}
}
return safecopy(dst, src, atEOF, nDst, nSrc, len(t))
}
// replace copies a replacement into the dst buffer and advances nDst and nSrc.
func replace(dst []byte, nDst, nSrc *int, replacement []byte, advance int) error {
if len(dst[*nDst:]) < len(replacement) {
return transform.ErrShortDst
}
n := copy(dst[*nDst:], replacement)
*nDst += n
*nSrc += advance
return nil
}
// copy copies data from src to dst for length n and advances nDst and nSrc.
func copyn(dst, src []byte, nDst, nSrc *int, n int) error {
copied := copy(dst[*nDst:], src[*nSrc:*nSrc+n])
*nDst += copied
*nSrc += copied
if copied < n {
return transform.ErrShortDst
}
return nil
}
// safecopy copies the remaining data minus that of the token size, preventing
// the accidental copy of the beginning of a token that should be replaced. If
// atEOF is true, the full remaining data is copied.
func safecopy(dst, src []byte, atEOF bool, nDst, nSrc int, tokenSize int) (int, int, error) {
var err error
remaining := len(src[nSrc:])
if !atEOF {
// copy either:
// - up until the last safe token if any, or
// - up to our data length minus tokenSize,
// whichever has the higher index position.
idx := bytes.LastIndexAny(src[nSrc:], safeTokens)
remaining -= tokenSize
if idx+1 > remaining {
remaining = idx + 1
}
err = transform.ErrShortSrc
}
if remaining > 0 {
err := copyn(dst, src, &nDst, &nSrc, remaining)
if err != nil {
return nDst, nSrc, err
}
}
return nDst, nSrc, err
}
package url_helpers
import "net/url"
func CleanURL(value string) (ret string) {
u, err := url.Parse(value)
if err != nil {
return
}
u.User = nil
u.RawQuery = ""
u.Fragment = ""
return u.String()
}
package url_helpers
import (
"regexp"
)
var scrubRegexp = regexp.MustCompile(
`(?im)([\?&]((?:private|authenticity|rss)[\-_]token)|X-AMZ-Signature|X-AMZ-Credential)=[^& ]*`,
)
// ScrubSecrets replaces the content of any sensitive query string parameters
// in a URL with `[FILTERED]`
func ScrubSecrets(url string) string {
return scrubRegexp.ReplaceAllString(url, "$1=[FILTERED]")
}
package auth_methods
import (
"fmt"
)
type MissingRequiredConfigurationKeyError struct {
key string
}
func NewMissingRequiredConfigurationKeyError(key string) *MissingRequiredConfigurationKeyError {
return &MissingRequiredConfigurationKeyError{
key: key,
}
}
func (e *MissingRequiredConfigurationKeyError) Error() string {
return fmt.Sprintf("missing required auth method configuration key %q", e.key)
}
func (e *MissingRequiredConfigurationKeyError) Is(err error) bool {
eerr, ok := err.(*MissingRequiredConfigurationKeyError)
if !ok {
return false
}
return eerr.key == e.key
}
type Data map[string]interface{}
func (d Data) Filter(requiredFields []string, allowedFields []string) (Data, error) {
for _, required := range requiredFields {
_, ok := d[required]
if !ok {
return nil, NewMissingRequiredConfigurationKeyError(required)
}
}
newData := make(Data)
for _, allowed := range allowedFields {
value, ok := d[allowed]
if !ok {
continue
}
newData[allowed] = value
}
return newData, nil
}
package jwt
import (
"fmt"
"path"
"gitlab.com/gitlab-org/gitlab-runner/helpers/vault"
"gitlab.com/gitlab-org/gitlab-runner/helpers/vault/auth_methods"
)
const methodName = "jwt"
const (
jwtKey = "jwt"
roleKey = "role"
)
var (
requiredPayloadFields = []string{
jwtKey,
}
allowedPayloadFields = []string{
jwtKey,
roleKey,
}
)
type method struct {
path string
data map[string]interface{}
token string
}
func NewMethod(path string, data auth_methods.Data) (vault.AuthMethod, error) {
newData, err := data.Filter(requiredPayloadFields, allowedPayloadFields)
if err != nil {
return nil, fmt.Errorf("filtering auth method configuration: %w", err)
}
a := &method{
path: path,
data: newData,
}
return a, nil
}
func (a *method) Name() string {
return methodName
}
func (a *method) Authenticate(client vault.Client) error {
authPath := path.Join("auth", a.path, "login")
authPayload := a.data
result, err := client.Write(authPath, authPayload)
if err != nil {
return fmt.Errorf("writing to Vault: %w", err)
}
token, err := result.TokenID()
if err != nil {
return fmt.Errorf("getting token from the authentication response: %w", err)
}
a.token = token
return nil
}
func (a *method) Token() string {
return a.token
}
func init() {
auth_methods.MustRegisterFactory(methodName, NewMethod)
}
package auth_methods
import (
"fmt"
"gitlab.com/gitlab-org/gitlab-runner/helpers/vault"
"gitlab.com/gitlab-org/gitlab-runner/helpers/vault/internal/registry"
)
type Factory func(path string, data Data) (vault.AuthMethod, error)
var factoriesRegistry = registry.New("auth method")
func MustRegisterFactory(authName string, factory Factory) {
err := factoriesRegistry.Register(authName, factory)
if err != nil {
panic(fmt.Sprintf("registering factory: %v", err))
}
}
func GetFactory(authName string) (Factory, error) {
factory, err := factoriesRegistry.Get(authName)
if err != nil {
return nil, err
}
return factory.(Factory), nil
}
package vault
import (
"errors"
"fmt"
"github.com/hashicorp/vault/api"
)
type Client interface {
Authenticate(auth AuthMethod) error
Write(path string, data map[string]interface{}) (Result, error)
Read(path string) (Result, error)
Delete(path string) error
}
type defaultClient struct {
internal apiClient
}
type apiClient interface {
Sys() apiClientSys
Logical() apiClientLogical
SetToken(v string)
}
type apiClientSys interface {
Health() (*api.HealthResponse, error)
}
type apiClientLogical interface {
Write(path string, data map[string]interface{}) (*api.Secret, error)
Read(path string) (*api.Secret, error)
Delete(path string) (*api.Secret, error)
}
type apiClientAdapter struct {
c *api.Client
}
func (c *apiClientAdapter) Sys() apiClientSys {
return c.c.Sys()
}
func (c *apiClientAdapter) Logical() apiClientLogical {
return c.c.Logical()
}
func (c *apiClientAdapter) SetToken(v string) {
c.c.SetToken(v)
}
var (
ErrVaultServerNotReady = errors.New("not initialized or sealed Vault server")
newAPIClient = func(config *api.Config) (apiClient, error) {
c, err := api.NewClient(config)
if err != nil {
return nil, err
}
return &apiClientAdapter{c: c}, nil
}
)
func NewClient(URL string) (Client, error) {
config := &api.Config{
Address: URL,
}
client, err := newAPIClient(config)
if err != nil {
return nil, fmt.Errorf("creating new Vault client: %w", unwrapAPIResponseError(err))
}
healthResp, err := client.Sys().Health()
if err != nil {
return nil, fmt.Errorf("checking Vault server health: %w", unwrapAPIResponseError(err))
}
if !healthResp.Initialized || healthResp.Sealed {
return nil, ErrVaultServerNotReady
}
c := &defaultClient{
internal: client,
}
return c, nil
}
func (c *defaultClient) Authenticate(auth AuthMethod) error {
err := auth.Authenticate(c)
if err != nil {
return fmt.Errorf("authenticating Vault client: %w", err)
}
c.internal.SetToken(auth.Token())
return nil
}
func (c *defaultClient) Write(path string, data map[string]interface{}) (Result, error) {
secret, err := c.internal.Logical().Write(path, data)
return newResult(secret), unwrapAPIResponseError(err)
}
func (c *defaultClient) Read(path string) (Result, error) {
secret, err := c.internal.Logical().Read(path)
return newResult(secret), unwrapAPIResponseError(err)
}
func (c *defaultClient) Delete(path string) error {
_, err := c.internal.Logical().Delete(path)
return unwrapAPIResponseError(err)
}
package registry
import (
"fmt"
)
type FactoryAlreadyRegisteredError struct {
factoryType string
factoryName string
}
func NewFactoryAlreadyRegisteredError(factoryType string, factoryName string) *FactoryAlreadyRegisteredError {
return &FactoryAlreadyRegisteredError{
factoryType: factoryType,
factoryName: factoryName,
}
}
func (e *FactoryAlreadyRegisteredError) Error() string {
return fmt.Sprintf("factory for %s %q already registered", e.factoryType, e.factoryName)
}
func (e *FactoryAlreadyRegisteredError) Is(err error) bool {
eerr, ok := err.(*FactoryAlreadyRegisteredError)
if !ok {
return false
}
return eerr.factoryName == e.factoryName
}
type FactoryNotRegisteredError struct {
factoryType string
factoryName string
}
func NewFactoryNotRegisteredError(factoryType string, factoryName string) *FactoryNotRegisteredError {
return &FactoryNotRegisteredError{
factoryType: factoryType,
factoryName: factoryName,
}
}
func (e *FactoryNotRegisteredError) Error() string {
return fmt.Sprintf("factory for %s %q is not registered", e.factoryType, e.factoryName)
}
func (e *FactoryNotRegisteredError) Is(err error) bool {
eerr, ok := err.(*FactoryNotRegisteredError)
if !ok {
return false
}
return eerr.factoryName == e.factoryName
}
type Registry interface {
Register(factoryName string, factory interface{}) error
Get(factoryName string) (interface{}, error)
}
type factoryRegistry struct {
factoryType string
store map[string]interface{}
}
func (r factoryRegistry) Register(factoryName string, factory interface{}) error {
_, ok := r.store[factoryName]
if ok {
return NewFactoryAlreadyRegisteredError(r.factoryType, factoryName)
}
r.store[factoryName] = factory
return nil
}
func (r factoryRegistry) Get(factoryName string) (interface{}, error) {
factory, ok := r.store[factoryName]
if !ok {
return nil, NewFactoryNotRegisteredError(r.factoryType, factoryName)
}
return factory, nil
}
func New(factoryType string) Registry {
return &factoryRegistry{
factoryType: factoryType,
store: make(map[string]interface{}),
}
}
package vault
import (
"errors"
"github.com/hashicorp/vault/api"
)
type Result interface {
Data() map[string]interface{}
TokenID() (string, error)
}
var ErrNoResult = errors.New("no result from Vault")
type secretResult struct {
inner *api.Secret
}
func newResult(secret *api.Secret) Result {
return &secretResult{
inner: secret,
}
}
func (r *secretResult) Data() map[string]interface{} {
if r.inner == nil {
return nil
}
return r.inner.Data
}
func (r *secretResult) TokenID() (string, error) {
if r.inner == nil {
return "", ErrNoResult
}
return r.inner.TokenID()
}
package kv_v1
import (
"fmt"
"path"
"gitlab.com/gitlab-org/gitlab-runner/helpers/vault"
"gitlab.com/gitlab-org/gitlab-runner/helpers/vault/secret_engines"
)
const engineName = "kv-v1"
type engine struct {
client vault.Client
path string
}
func NewEngine(client vault.Client, path string) vault.SecretEngine {
return &engine{
client: client,
path: path,
}
}
func (e *engine) EngineName() string {
return engineName
}
func (e *engine) Get(path string) (map[string]interface{}, error) {
secret, err := e.client.Read(e.fullPath(path))
if err != nil {
return nil, fmt.Errorf("reading from Vault: %w", err)
}
return secret.Data(), nil
}
func (e *engine) fullPath(p string) string {
return path.Join(e.path, p)
}
func (e *engine) Put(path string, data map[string]interface{}) error {
_, err := e.client.Write(e.fullPath(path), data)
if err != nil {
return fmt.Errorf("writing to Vault: %w", err)
}
return nil
}
func (e *engine) Delete(path string) error {
err := e.client.Delete(e.fullPath(path))
if err != nil {
return fmt.Errorf("deleting from Vault: %w", err)
}
return nil
}
func init() {
secret_engines.MustRegisterFactory(engineName, NewEngine)
}
package kv_v2
import (
"fmt"
"path"
"gitlab.com/gitlab-org/gitlab-runner/helpers/vault"
"gitlab.com/gitlab-org/gitlab-runner/helpers/vault/secret_engines"
)
const engineName = "kv-v2"
type engine struct {
client vault.Client
path string
}
func NewEngine(client vault.Client, path string) vault.SecretEngine {
return &engine{
client: client,
path: path,
}
}
func (e *engine) EngineName() string {
return engineName
}
func (e *engine) Get(path string) (map[string]interface{}, error) {
secret, err := e.client.Read(e.dataPath(path))
if err != nil {
return nil, fmt.Errorf("reading from Vault: %w", err)
}
if secret == nil {
return nil, nil
}
data := secret.Data()
if data == nil {
return nil, nil
}
_, ok := data["data"]
if !ok {
return nil, nil
}
return data["data"].(map[string]interface{}), nil
}
func (e *engine) dataPath(p string) string {
return path.Join(e.path, "data", p)
}
func (e *engine) Put(path string, data map[string]interface{}) error {
dataWrapper := map[string]interface{}{
"data": data,
}
_, err := e.client.Write(e.dataPath(path), dataWrapper)
if err != nil {
return fmt.Errorf("writing to Vault: %w", err)
}
return nil
}
func (e *engine) Delete(path string) error {
err := e.client.Delete(e.metadataPath(path))
if err != nil {
return fmt.Errorf("deleting from Vault: %w", err)
}
return nil
}
func (e *engine) metadataPath(p string) string {
return path.Join(e.path, "metadata", p)
}
func init() {
secret_engines.MustRegisterFactory(engineName, NewEngine)
}
package secret_engines
import (
"fmt"
"gitlab.com/gitlab-org/gitlab-runner/helpers/vault"
)
type OperationType string
const (
getOperation OperationType = "get"
putOperation OperationType = "put"
deleteOperation OperationType = "delete"
)
type OperationNotSupportedError struct {
secretEngineName string
operationType OperationType
}
func NewUnsupportedGetOperationErr(engine vault.SecretEngine) *OperationNotSupportedError {
return newErrOperationNotSupported(engine, getOperation)
}
func NewUnsupportedPutOperationErr(engine vault.SecretEngine) *OperationNotSupportedError {
return newErrOperationNotSupported(engine, putOperation)
}
func NewUnsupportedDeleteOperationErr(engine vault.SecretEngine) *OperationNotSupportedError {
return newErrOperationNotSupported(engine, deleteOperation)
}
func newErrOperationNotSupported(engine vault.SecretEngine, operationType OperationType) *OperationNotSupportedError {
return &OperationNotSupportedError{
secretEngineName: engine.EngineName(),
operationType: operationType,
}
}
func (e *OperationNotSupportedError) Error() string {
return fmt.Sprintf("operation %q for secret engine %q is not supported", e.operationType, e.secretEngineName)
}
func (e *OperationNotSupportedError) Is(err error) bool {
eerr, ok := err.(*OperationNotSupportedError)
if !ok {
return false
}
return eerr.secretEngineName == e.secretEngineName && eerr.operationType == e.operationType
}
package secret_engines
import (
"fmt"
"gitlab.com/gitlab-org/gitlab-runner/helpers/vault"
"gitlab.com/gitlab-org/gitlab-runner/helpers/vault/internal/registry"
)
type Factory func(client vault.Client, path string) vault.SecretEngine
var factoriesRegistry = registry.New("secret engine")
func MustRegisterFactory(engineName string, factory Factory) {
err := factoriesRegistry.Register(engineName, factory)
if err != nil {
panic(fmt.Sprintf("registering factory: %v", err))
}
}
func GetFactory(engineName string) (Factory, error) {
factory, err := factoriesRegistry.Get(engineName)
if err != nil {
return nil, err
}
return factory.(Factory), nil
}
package service
import (
"fmt"
"gitlab.com/gitlab-org/gitlab-runner/helpers/vault"
"gitlab.com/gitlab-org/gitlab-runner/helpers/vault/auth_methods"
_ "gitlab.com/gitlab-org/gitlab-runner/helpers/vault/auth_methods/jwt" // register auth method
"gitlab.com/gitlab-org/gitlab-runner/helpers/vault/secret_engines"
_ "gitlab.com/gitlab-org/gitlab-runner/helpers/vault/secret_engines/kv_v1" // register secret engine
_ "gitlab.com/gitlab-org/gitlab-runner/helpers/vault/secret_engines/kv_v2" // register secret engine
)
type Auth interface {
AuthName() string
AuthPath() string
AuthData() auth_methods.Data
}
type Engine interface {
EngineName() string
EnginePath() string
}
type Secret interface {
SecretPath() string
SecretField() string
}
type Vault interface {
GetField(engineDetails Engine, secretDetails Secret) (interface{}, error)
Put(engineDetails Engine, secretDetails Secret, data map[string]interface{}) error
Delete(engineDetails Engine, secretDetails Secret) error
}
type defaultVault struct {
client vault.Client
}
var newVaultClient = vault.NewClient
func NewVault(url string, auth Auth) (Vault, error) {
v := new(defaultVault)
err := v.initialize(url, auth)
if err != nil {
return nil, fmt.Errorf("initializing Vault service: %w", err)
}
return v, nil
}
func (v *defaultVault) initialize(url string, auth Auth) error {
err := v.prepareAuthenticatedClient(url, auth)
if err != nil {
return fmt.Errorf("preparing authenticated client: %w", err)
}
return nil
}
func (v *defaultVault) prepareAuthenticatedClient(url string, authDetails Auth) error {
client, err := newVaultClient(url)
if err != nil {
return err
}
auth, err := v.prepareAuthMethodAdapter(authDetails)
if err != nil {
return err
}
err = client.Authenticate(auth)
if err != nil {
return err
}
v.client = client
return nil
}
func (v *defaultVault) prepareAuthMethodAdapter(authDetails Auth) (vault.AuthMethod, error) {
authFactory, err := auth_methods.GetFactory(authDetails.AuthName())
if err != nil {
return nil, fmt.Errorf("initializing auth method factory: %w", err)
}
auth, err := authFactory(authDetails.AuthPath(), authDetails.AuthData())
if err != nil {
return nil, fmt.Errorf("initializing auth method adapter: %w", err)
}
return auth, nil
}
func (v *defaultVault) GetField(engineDetails Engine, secretDetails Secret) (interface{}, error) {
engine, err := v.getSecretEngine(engineDetails)
if err != nil {
return nil, err
}
secret, err := engine.Get(secretDetails.SecretPath())
if err != nil {
return nil, fmt.Errorf("reading secret: %w", err)
}
field := secretDetails.SecretField()
for key, data := range secret {
if key != field {
continue
}
return data, nil
}
return nil, nil
}
func (v *defaultVault) getSecretEngine(engineDetails Engine) (vault.SecretEngine, error) {
engineFactory, err := secret_engines.GetFactory(engineDetails.EngineName())
if err != nil {
return nil, fmt.Errorf("requesting SecretEngine factory: %w", err)
}
engine := engineFactory(v.client, engineDetails.EnginePath())
return engine, nil
}
func (v *defaultVault) Put(engineDetails Engine, secretDetails Secret, data map[string]interface{}) error {
engine, err := v.getSecretEngine(engineDetails)
if err != nil {
return err
}
err = engine.Put(secretDetails.SecretPath(), data)
if err != nil {
return fmt.Errorf("writing secret: %w", err)
}
return nil
}
func (v *defaultVault) Delete(engineDetails Engine, secretDetails Secret) error {
engine, err := v.getSecretEngine(engineDetails)
if err != nil {
return err
}
err = engine.Delete(secretDetails.SecretPath())
if err != nil {
return fmt.Errorf("deleting secret: %w", err)
}
return nil
}
package vault
import (
"errors"
"fmt"
"strings"
"github.com/hashicorp/vault/api"
)
type unwrappedAPIResponseError struct {
statusCode int
apiErrors string
}
func newUnwrappedAPIResponseError(statusCode int, errors []string) *unwrappedAPIResponseError {
return &unwrappedAPIResponseError{
statusCode: statusCode,
apiErrors: strings.Join(errors, ", "),
}
}
func (e *unwrappedAPIResponseError) Error() string {
return fmt.Sprintf("api error: status code %d: %s", e.statusCode, e.apiErrors)
}
func (e *unwrappedAPIResponseError) Is(err error) bool {
eerr, ok := err.(*unwrappedAPIResponseError)
if !ok {
return false
}
return eerr.statusCode == e.statusCode && eerr.apiErrors == e.apiErrors
}
func unwrapAPIResponseError(err error) error {
if err == nil {
return nil
}
apiErr := new(api.ResponseError)
if !errors.As(err, &apiErr) {
return err
}
return newUnwrappedAPIResponseError(apiErr.StatusCode, apiErr.Errors)
}
package virtualbox
import (
"bytes"
"errors"
"fmt"
"net"
"os"
"os/exec"
"regexp"
"strings"
"time"
"github.com/sirupsen/logrus"
)
type StatusType string
const (
NotFound StatusType = "notfound"
PoweredOff StatusType = "poweroff"
Saved StatusType = "saved"
Teleported StatusType = "teleported"
Aborted StatusType = "aborted"
Running StatusType = "running"
Paused StatusType = "paused"
Stuck StatusType = "gurumeditation"
Teleporting StatusType = "teleporting"
LiveSnapshotting StatusType = "livesnapshotting"
Starting StatusType = "starting"
Stopping StatusType = "stopping"
Saving StatusType = "saving"
Restoring StatusType = "restoring"
TeleportingPausedVM StatusType = "teleportingpausedvm"
TeleportingIn StatusType = "teleportingin"
FaultTolerantSyncing StatusType = "faulttolerantsyncing"
DeletingSnapshotOnline StatusType = "deletingsnapshotlive"
DeletingSnapshotPaused StatusType = "deletingsnapshotlivepaused"
OnlineSnapshotting StatusType = "onlinesnapshotting"
RestoringSnapshot StatusType = "restoringsnapshot"
DeletingSnapshot StatusType = "deletingsnapshot"
SettingUp StatusType = "settingup"
Snapshotting StatusType = "snapshotting"
Unknown StatusType = "unknown"
// TODO: update as new VM states are added
)
func IsStatusOnlineOrTransient(vmStatus StatusType) bool {
switch vmStatus {
case Running,
Paused,
Stuck,
Teleporting,
LiveSnapshotting,
Starting,
Stopping,
Saving,
Restoring,
TeleportingPausedVM,
TeleportingIn,
FaultTolerantSyncing,
DeletingSnapshotOnline,
DeletingSnapshotPaused,
OnlineSnapshotting,
RestoringSnapshot,
DeletingSnapshot,
SettingUp,
Snapshotting:
return true
}
return false
}
func VboxManageOutput(exe string, args ...string) (string, error) {
var stdout, stderr bytes.Buffer
logrus.Debugf("Executing VBoxManageOutput: %#v", args)
cmd := exec.Command(exe, args...)
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
stderrString := strings.TrimSpace(stderr.String())
if _, ok := err.(*exec.ExitError); ok {
err = fmt.Errorf("VBoxManageOutput error: %s", stderrString)
}
return stdout.String(), err
}
func VBoxManage(args ...string) (string, error) {
return VboxManageOutput("vboxmanage", args...)
}
func Version() (string, error) {
version, err := VBoxManage("--version")
if err != nil {
return "", err
}
return strings.TrimSpace(version), nil
}
func FindSSHPort(vmName string) (port string, err error) {
info, err := VBoxManage("showvminfo", vmName)
if err != nil {
return
}
portRe := regexp.MustCompile(`guestssh.*host port = (\d+)`)
sshPort := portRe.FindStringSubmatch(info)
if len(sshPort) >= 2 {
port = sshPort[1]
} else {
err = errors.New("failed to find guestssh port")
}
return
}
func Exist(vmName string) bool {
_, err := VBoxManage("showvminfo", vmName)
return err == nil
}
func CreateOsVM(vmName string, templateName string, templateSnapshot string, baseFolder string) error {
args := []string{"clonevm", vmName, "--mode", "machine", "--name", templateName, "--register"}
if templateSnapshot != "" {
args = append(args, "--snapshot", templateSnapshot, "--options", "link")
}
if baseFolder != "" {
args = append(args, "--basefolder", baseFolder)
}
_, err := VBoxManage(args...)
return err
}
func isPortUnassigned(testPort string, usedPorts [][]string) bool {
for _, port := range usedPorts {
if testPort == port[1] {
return false
}
}
return true
}
func getUsedVirtualBoxPorts() (usedPorts [][]string, err error) {
output, err := VBoxManage("list", "vms", "-l")
if err != nil {
return
}
allPortsRe := regexp.MustCompile(`host port = (\d+)`)
usedPorts = allPortsRe.FindAllStringSubmatch(output, -1)
return
}
func allocatePort(handler func(port string) error) (port string, err error) {
ln, err := net.Listen("tcp", ":0")
if err != nil {
logrus.Debugln("VirtualBox ConfigureSSH:", err)
return
}
defer func() { _ = ln.Close() }()
usedPorts, err := getUsedVirtualBoxPorts()
if err != nil {
logrus.Debugln("VirtualBox ConfigureSSH:", err)
return
}
addressElements := strings.Split(ln.Addr().String(), ":")
port = addressElements[len(addressElements)-1]
if isPortUnassigned(port, usedPorts) {
err = handler(port)
} else {
err = os.ErrExist
}
return
}
func ConfigureSSH(vmName string, vmSSHPort string) (port string, err error) {
for {
port, err = allocatePort(
func(port string) error {
rule := fmt.Sprintf("guestssh,tcp,127.0.0.1,%s,,%s", port, vmSSHPort)
_, err = VBoxManage("modifyvm", vmName, "--natpf1", rule)
return err
},
)
if err == nil || err != os.ErrExist {
return
}
}
}
func CreateSnapshot(vmName string, snapshotName string) error {
_, err := VBoxManage("snapshot", vmName, "take", snapshotName)
return err
}
func RevertToSnapshot(vmName string) error {
_, err := VBoxManage("snapshot", vmName, "restorecurrent")
return err
}
func matchSnapshotName(snapshotName string, snapshotList string) bool {
snapshotRe := regexp.MustCompile(
fmt.Sprintf(`(?m)^Snapshot(Name|UUID)[^=]*="(%s)"\r?$`, regexp.QuoteMeta(snapshotName)),
)
snapshot := snapshotRe.FindStringSubmatch(snapshotList)
return snapshot != nil
}
func HasSnapshot(vmName string, snapshotName string) bool {
output, err := VBoxManage("snapshot", vmName, "list", "--machinereadable")
if err != nil {
return false
}
return matchSnapshotName(snapshotName, output)
}
func matchCurrentSnapshotName(snapshotList string) []string {
snapshotRe := regexp.MustCompile(`(?m)^CurrentSnapshotName="([^"]*)"\r?$`)
return snapshotRe.FindStringSubmatch(snapshotList)
}
func GetCurrentSnapshot(vmName string) (string, error) {
output, err := VBoxManage("snapshot", vmName, "list", "--machinereadable")
if err != nil {
return "", err
}
snapshot := matchCurrentSnapshotName(output)
if snapshot == nil {
return "", errors.New("failed to match current snapshot name")
}
return snapshot[1], nil
}
func Start(vmName string) error {
_, err := VBoxManage("startvm", vmName, "--type", "headless")
return err
}
func Kill(vmName string) error {
_, err := VBoxManage("controlvm", vmName, "poweroff")
return err
}
func Delete(vmName string) error {
_, err := VBoxManage("unregistervm", vmName, "--delete")
return err
}
func Status(vmName string) (StatusType, error) {
output, err := VBoxManage("showvminfo", vmName, "--machinereadable")
statusRe := regexp.MustCompile(`VMState="(\w+)"`)
status := statusRe.FindStringSubmatch(output)
if err != nil {
return NotFound, err
}
return StatusType(status[1]), nil
}
func WaitForStatus(vmName string, vmStatus StatusType, seconds int) error {
var status StatusType
var err error
for i := 0; i < seconds; i++ {
status, err = Status(vmName)
if err != nil {
return err
}
if status == vmStatus {
return nil
}
time.Sleep(time.Second)
}
return errors.New("VM " + vmName + " is in " + string(status) + " where it should be in " + string(vmStatus))
}
func Unregister(vmName string) error {
_, err := VBoxManage("unregistervm", vmName)
return err
}
package log
import (
"fmt"
"os"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)
const (
FormatRunner = "runner"
FormatText = "text"
FormatJSON = "json"
)
var (
configuration = NewConfig(logrus.StandardLogger())
logFlags = []cli.Flag{
cli.BoolFlag{
Name: "debug",
Usage: "debug mode",
EnvVar: "DEBUG",
},
cli.StringFlag{
Name: "log-format",
Usage: "Choose log format (options: runner, text, json)",
EnvVar: "LOG_FORMAT",
},
cli.StringFlag{
Name: "log-level, l",
Usage: "Log level (options: debug, info, warn, error, fatal, panic)",
EnvVar: "LOG_LEVEL",
},
}
formats = map[string]logrus.Formatter{
FormatRunner: new(RunnerTextFormatter),
FormatText: new(logrus.TextFormatter),
FormatJSON: new(logrus.JSONFormatter),
}
)
func formatNames() []string {
formatNames := make([]string, 0)
for name := range formats {
formatNames = append(formatNames, name)
}
return formatNames
}
type Config struct {
logger *logrus.Logger
level logrus.Level
format logrus.Formatter
levelSetWithCli bool
formatSetWithCli bool
goroutinesDumpStopCh chan bool
}
func (l *Config) IsLevelSetWithCli() bool {
return l.levelSetWithCli
}
func (l *Config) IsFormatSetWithCli() bool {
return l.formatSetWithCli
}
func (l *Config) handleCliCtx(cliCtx *cli.Context) error {
if cliCtx.IsSet("log-level") || cliCtx.IsSet("l") {
err := l.SetLevel(cliCtx.String("log-level"))
if err != nil {
return err
}
l.levelSetWithCli = true
}
if cliCtx.Bool("debug") {
l.level = logrus.DebugLevel
l.levelSetWithCli = true
}
if cliCtx.IsSet("log-format") {
err := l.SetFormat(cliCtx.String("log-format"))
if err != nil {
return err
}
l.formatSetWithCli = true
}
l.ReloadConfiguration()
return nil
}
func (l *Config) SetLevel(levelString string) error {
level, err := logrus.ParseLevel(levelString)
if err != nil {
return fmt.Errorf("failed to parse log level: %w", err)
}
l.level = level
return nil
}
func (l *Config) SetFormat(format string) error {
formatter, ok := formats[format]
if !ok {
return fmt.Errorf("unknown log format %q, expected one of: %v", l.format, formatNames())
}
l.format = formatter
return nil
}
func (l *Config) ReloadConfiguration() {
l.logger.SetFormatter(l.format)
l.logger.SetLevel(l.level)
if l.level == logrus.DebugLevel {
l.enableGoroutinesDump()
} else {
l.disableGoroutinesDump()
}
}
func (l *Config) enableGoroutinesDump() {
if l.goroutinesDumpStopCh != nil {
return
}
l.goroutinesDumpStopCh = make(chan bool)
watchForGoroutinesDump(l.logger, l.goroutinesDumpStopCh)
}
func (l *Config) disableGoroutinesDump() {
if l.goroutinesDumpStopCh == nil {
return
}
close(l.goroutinesDumpStopCh)
l.goroutinesDumpStopCh = nil
}
func NewConfig(logger *logrus.Logger) *Config {
return &Config{
logger: logger,
level: logrus.InfoLevel,
format: new(RunnerTextFormatter),
}
}
func Configuration() *Config {
return configuration
}
func ConfigureLogging(app *cli.App) {
app.Flags = append(app.Flags, logFlags...)
appBefore := app.Before
app.Before = func(cliCtx *cli.Context) error {
Configuration().logger.SetOutput(os.Stderr)
err := Configuration().handleCliCtx(cliCtx)
if err != nil {
logrus.WithError(err).Fatal("Error while setting up logging configuration")
}
if appBefore != nil {
return appBefore(cliCtx)
}
return nil
}
}
// +build darwin dragonfly freebsd linux netbsd openbsd
package log
import (
"os"
"os/signal"
"runtime"
"syscall"
"github.com/sirupsen/logrus"
)
func watchForGoroutinesDump(logger *logrus.Logger, stopCh chan bool) (chan bool, chan bool) {
dumpedCh := make(chan bool)
finishedCh := make(chan bool)
dumpStacksCh := make(chan os.Signal, 1)
// On USR1 dump stacks of all go routines
signal.Notify(dumpStacksCh, syscall.SIGUSR1)
go func() {
for {
select {
case <-dumpStacksCh:
buf := make([]byte, 1<<20)
len := runtime.Stack(buf, true)
logger.Printf("=== received SIGUSR1 ===\n*** goroutine dump...\n%s\n*** end\n", buf[0:len])
nonBlockingSend(dumpedCh, true)
case <-stopCh:
close(finishedCh)
return
}
}
}()
return dumpedCh, finishedCh
}
func nonBlockingSend(ch chan bool, value bool) {
select {
case ch <- value:
default:
}
}
package log
import (
"bytes"
"fmt"
"sort"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
)
type RunnerTextFormatter struct {
// Force disabling colors.
DisableColors bool
// The fields are sorted by default for a consistent output. For applications
// that log extremely frequently and don't use the JSON formatter this may not
// be desired.
DisableSorting bool
}
func (f *RunnerTextFormatter) Format(entry *logrus.Entry) ([]byte, error) {
b := new(bytes.Buffer)
f.printColored(b, entry)
b.WriteByte('\n')
return b.Bytes(), nil
}
func (f *RunnerTextFormatter) printColored(b *bytes.Buffer, entry *logrus.Entry) {
levelColor, resetColor, levelPrefix := f.getColorsAndPrefix(entry)
indentLength := 50 - len(levelPrefix)
fmt.Fprintf(b, "%s%s%-*s%s ", levelColor, levelPrefix, indentLength, entry.Message, resetColor)
for _, k := range f.prepareKeys(entry) {
v := entry.Data[k]
fmt.Fprintf(b, " %s%s%s=%v", levelColor, k, resetColor, v)
}
}
func (f *RunnerTextFormatter) getColorsAndPrefix(entry *logrus.Entry) (string, string, string) {
definitions := map[logrus.Level]struct {
color string
prefix string
}{
logrus.DebugLevel: {
color: helpers.ANSI_BOLD_WHITE,
},
logrus.WarnLevel: {
color: helpers.ANSI_YELLOW,
prefix: "WARNING: ",
},
logrus.ErrorLevel: {
color: helpers.ANSI_BOLD_RED,
prefix: "ERROR: ",
},
logrus.FatalLevel: {
color: helpers.ANSI_BOLD_RED,
prefix: "FATAL: ",
},
logrus.PanicLevel: {
color: helpers.ANSI_BOLD_RED,
prefix: "PANIC: ",
},
}
color := ""
prefix := ""
definition, ok := definitions[entry.Level]
if ok {
if definition.color != "" {
color = definition.color
}
if definition.prefix != "" {
prefix = definition.prefix
}
}
if f.DisableColors {
return "", "", prefix
}
return color, helpers.ANSI_RESET, prefix
}
func (f *RunnerTextFormatter) prepareKeys(entry *logrus.Entry) []string {
keys := make([]string, 0, len(entry.Data))
for k := range entry.Data {
keys = append(keys, k)
}
if !f.DisableSorting {
sort.Strings(keys)
}
return keys
}
func SetRunnerFormatter() {
logrus.SetFormatter(new(RunnerTextFormatter))
}
package log
import (
"github.com/sirupsen/logrus"
url_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/url"
)
type SecretsCleanupHook struct{}
func (s *SecretsCleanupHook) Levels() []logrus.Level {
return logrus.AllLevels
}
func (s *SecretsCleanupHook) Fire(entry *logrus.Entry) error {
entry.Message = url_helpers.ScrubSecrets(entry.Message)
return nil
}
func AddSecretsCleanupLogHook(logger *logrus.Logger) {
if logger == nil {
logger = logrus.StandardLogger()
}
logger.AddHook(new(SecretsCleanupHook))
}
package log
import (
"github.com/kardianos/service"
"github.com/sirupsen/logrus"
)
type systemLogger interface {
service.Logger
}
type systemService interface {
service.Service
}
type SystemServiceLogHook struct {
systemLogger
Level logrus.Level
}
func (s *SystemServiceLogHook) Levels() []logrus.Level {
return []logrus.Level{
logrus.PanicLevel,
logrus.FatalLevel,
logrus.ErrorLevel,
logrus.WarnLevel,
logrus.InfoLevel,
}
}
func (s *SystemServiceLogHook) Fire(entry *logrus.Entry) error {
if entry.Level > s.Level {
return nil
}
msg, err := entry.String()
if err != nil {
return err
}
switch entry.Level {
case logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel:
return s.Error(msg)
case logrus.WarnLevel:
return s.Warning(msg)
case logrus.InfoLevel:
return s.Info(msg)
}
return nil
}
func SetSystemLogger(logrusLogger *logrus.Logger, svc systemService) {
logger, err := svc.SystemLogger(nil)
if err == nil {
hook := new(SystemServiceLogHook)
hook.systemLogger = logger
hook.Level = logrus.GetLevel()
logrusLogger.AddHook(hook)
} else {
logrusLogger.WithError(err).Error("Error while setting up the system logger")
}
}
package test
import (
"github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/test"
)
// NewHook will create a new global hook that can be used for tests after which
// it will remove when the returned function invoked.
//
// This shouldn't be used when you are writing a new package/structure, you
// should instead pass the logger to that struct and add the Hook to that struct
// only, try to avoid the global logger. This has multiple benefits, for example
// having that struct with specific logger settings that doesn't effect the
// logger in another part of the application. For example:
//
// type MyNewStruct struct {
// logger logrus.FieldLogger
// }
//
// The more hooks we add to the tests the more memory we are leaking.
func NewHook() (*test.Hook, func()) {
// Copy all the previous hooks so we revert back to that state.
oldHooks := logrus.LevelHooks{}
for level, hooks := range logrus.StandardLogger().Hooks {
oldHooks[level] = hooks
}
newHook := test.NewGlobal()
return newHook, func() {
logrus.StandardLogger().ReplaceHooks(oldHooks)
}
}
package main
import (
"os"
"path/filepath"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"gitlab.com/gitlab-org/gitlab-runner/common"
cli_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/cli"
"gitlab.com/gitlab-org/gitlab-runner/log"
_ "gitlab.com/gitlab-org/gitlab-runner/cache/azure"
_ "gitlab.com/gitlab-org/gitlab-runner/cache/gcs"
_ "gitlab.com/gitlab-org/gitlab-runner/cache/s3"
_ "gitlab.com/gitlab-org/gitlab-runner/commands"
_ "gitlab.com/gitlab-org/gitlab-runner/commands/helpers"
_ "gitlab.com/gitlab-org/gitlab-runner/executors/custom"
_ "gitlab.com/gitlab-org/gitlab-runner/executors/docker"
_ "gitlab.com/gitlab-org/gitlab-runner/executors/docker/machine"
_ "gitlab.com/gitlab-org/gitlab-runner/executors/kubernetes"
_ "gitlab.com/gitlab-org/gitlab-runner/executors/parallels"
_ "gitlab.com/gitlab-org/gitlab-runner/executors/shell"
_ "gitlab.com/gitlab-org/gitlab-runner/executors/ssh"
_ "gitlab.com/gitlab-org/gitlab-runner/executors/virtualbox"
_ "gitlab.com/gitlab-org/gitlab-runner/helpers/secrets/resolvers/vault"
_ "gitlab.com/gitlab-org/gitlab-runner/shells"
)
func main() {
defer func() {
if r := recover(); r != nil {
// log panics forces exit
if _, ok := r.(*logrus.Entry); ok {
os.Exit(1)
}
panic(r)
}
}()
app := cli.NewApp()
app.Name = filepath.Base(os.Args[0])
app.Usage = "a GitLab Runner"
app.Version = common.AppVersion.ShortLine()
cli.VersionPrinter = common.AppVersion.Printer
app.Authors = []cli.Author{
{
Name: "GitLab Inc.",
Email: "support@gitlab.com",
},
}
app.Commands = common.GetCommands()
app.CommandNotFound = func(context *cli.Context, command string) {
logrus.Fatalln("Command", command, "not found.")
}
cli_helpers.InitCli()
cli_helpers.LogRuntimePlatform(app)
cli_helpers.SetupCPUProfile(app)
cli_helpers.FixHOME(app)
cli_helpers.WarnOnBool(os.Args)
log.ConfigureLogging(app)
if err := app.Run(os.Args); err != nil {
logrus.Fatal(err)
}
}
package network
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"mime"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"time"
"github.com/jpillora/backoff"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/tls/ca_chain"
)
const jsonMimeType = "application/json"
type requestCredentials interface {
GetURL() string
GetToken() string
GetTLSCAFile() string
GetTLSCertFile() string
GetTLSKeyFile() string
}
var dialer = net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}
const (
backOffDelayMin = 100 * time.Millisecond
backOffDelayMax = 60 * time.Second
backOffDelayFactor = 2.0
backOffDelayJitter = true
)
type client struct {
http.Client
url *url.URL
caFile string
certFile string
keyFile string
caData []byte
updateTime time.Time
lastUpdate string
requestBackOffs map[string]*backoff.Backoff
lock sync.Mutex
requester requester
}
type ResponseTLSData struct {
CAChain string
CertFile string
KeyFile string
}
func (n *client) getLastUpdate() string {
return n.lastUpdate
}
func (n *client) setLastUpdate(headers http.Header) {
if lu := headers.Get("X-GitLab-Last-Update"); len(lu) > 0 {
n.lastUpdate = lu
}
}
func (n *client) ensureTLSConfig() {
// certificate got modified
if stat, err := os.Stat(n.caFile); err == nil && n.updateTime.Before(stat.ModTime()) {
n.Transport = nil
}
// client certificate got modified
if stat, err := os.Stat(n.certFile); err == nil && n.updateTime.Before(stat.ModTime()) {
n.Transport = nil
}
// client private key got modified
if stat, err := os.Stat(n.keyFile); err == nil && n.updateTime.Before(stat.ModTime()) {
n.Transport = nil
}
// create or update transport
if n.Transport == nil {
n.updateTime = time.Now()
n.createTransport()
}
}
func (n *client) addTLSCA(tlsConfig *tls.Config) {
// load TLS CA certificate
file := n.caFile
if file == "" {
return
}
logrus.Debugln("Trying to load", file, "...")
data, err := ioutil.ReadFile(file)
if err != nil {
if !os.IsNotExist(err) {
logrus.Errorln("Failed to load", n.caFile, err)
}
return
}
// SystemCertPool doesn't work on Windows: https://github.com/golang/go/issues/16736
pool, err := x509.SystemCertPool()
if err != nil && runtime.GOOS != "windows" {
logrus.Warningln("Failed to load system CertPool:", err)
}
if pool == nil {
pool = x509.NewCertPool()
}
if !pool.AppendCertsFromPEM(data) {
logrus.Errorln("Failed to parse PEM in", n.caFile)
return
}
tlsConfig.RootCAs = pool
n.caData = data
}
func (n *client) addTLSAuth(tlsConfig *tls.Config) {
if n.certFile == "" || n.keyFile == "" {
return
}
logrus.Debugln("Trying to load", n.certFile, "and", n.keyFile, "pair...")
// load TLS client keypair
certificate, err := tls.LoadX509KeyPair(n.certFile, n.keyFile)
if err != nil {
if !os.IsNotExist(err) {
logrus.Errorln("Failed to load", n.certFile, n.keyFile, err)
}
return
}
tlsConfig.Certificates = []tls.Certificate{certificate}
tlsConfig.BuildNameToCertificate()
}
func (n *client) createTransport() {
// create reference TLS config
tlsConfig := tls.Config{
MinVersion: tls.VersionTLS12,
}
n.addTLSCA(&tlsConfig)
n.addTLSAuth(&tlsConfig)
// create transport
n.Transport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: func(network, addr string) (net.Conn, error) {
logrus.Debugln("Dialing:", network, addr, "...")
return dialer.Dial(network, addr)
},
TLSClientConfig: &tlsConfig,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
ResponseHeaderTimeout: 10 * time.Minute,
}
n.Timeout = common.DefaultNetworkClientTimeout
}
func (n *client) ensureBackoff(method, uri string) *backoff.Backoff {
n.lock.Lock()
defer n.lock.Unlock()
key := fmt.Sprintf("%s_%s", method, uri)
if n.requestBackOffs[key] == nil {
n.requestBackOffs[key] = &backoff.Backoff{
Min: backOffDelayMin,
Max: backOffDelayMax,
Factor: backOffDelayFactor,
Jitter: backOffDelayJitter,
}
}
return n.requestBackOffs[key]
}
func (n *client) backoffRequired(res *http.Response) bool {
return res.StatusCode >= 400 && res.StatusCode < 600
}
func (n *client) checkBackoffRequest(req *http.Request, res *http.Response) {
backoffDelay := n.ensureBackoff(req.Method, req.RequestURI)
if n.backoffRequired(res) {
time.Sleep(backoffDelay.Duration())
} else {
backoffDelay.Reset()
}
}
func (n *client) do(
ctx context.Context,
uri, method string,
request io.Reader,
requestType string,
headers http.Header,
) (*http.Response, error) {
url, err := n.url.Parse(uri)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, method, url.String(), request)
if err != nil {
err = fmt.Errorf("failed to create NewRequest: %w", err)
return nil, err
}
if headers != nil {
req.Header = headers
}
if request != nil {
req.Header.Set("Content-Type", requestType)
req.Header.Set("User-Agent", common.AppVersion.UserAgent())
}
n.ensureTLSConfig()
res, err := n.requester.Do(req)
if err != nil {
return nil, err
}
n.checkBackoffRequest(req, res)
return res, nil
}
func (n *client) doJSON(
ctx context.Context,
uri, method string,
statusCode int,
request interface{},
response interface{},
) (int, string, *http.Response) {
var body io.Reader
if request != nil {
requestBody, err := json.Marshal(request)
if err != nil {
return -1, fmt.Sprintf("failed to marshal project object: %v", err), nil
}
body = bytes.NewReader(requestBody)
}
headers := make(http.Header)
if response != nil {
headers.Set("Accept", jsonMimeType)
}
res, err := n.do(ctx, uri, method, body, jsonMimeType, headers)
if err != nil {
return -1, err.Error(), nil
}
defer func() {
_, _ = io.Copy(ioutil.Discard, res.Body)
_ = res.Body.Close()
}()
if res.StatusCode == statusCode && response != nil {
isApplicationJSON, err := isResponseApplicationJSON(res)
if !isApplicationJSON {
return -1, err.Error(), nil
}
d := json.NewDecoder(res.Body)
err = d.Decode(response)
if err != nil {
return -1, fmt.Sprintf("Error decoding json payload %v", err), nil
}
}
n.setLastUpdate(res.Header)
return res.StatusCode, res.Status, res
}
func (n *client) getResponseTLSData(tls *tls.ConnectionState) (ResponseTLSData, error) {
TLSData := ResponseTLSData{
CertFile: n.certFile,
KeyFile: n.keyFile,
}
caChain, err := n.buildCAChain(tls)
if err != nil {
return TLSData, fmt.Errorf("couldn't build CA Chain: %w", err)
}
TLSData.CAChain = caChain
return TLSData, nil
}
func (n *client) buildCAChain(tls *tls.ConnectionState) (string, error) {
if len(n.caData) != 0 {
return string(n.caData), nil
}
if tls == nil {
return "", nil
}
builder := ca_chain.NewBuilder(logrus.StandardLogger())
err := builder.BuildChainFromTLSConnectionState(tls)
if err != nil {
return "", fmt.Errorf("error while fetching certificates from TLS ConnectionState: %w", err)
}
return builder.String(), nil
}
func isResponseApplicationJSON(res *http.Response) (result bool, err error) {
contentType := res.Header.Get("Content-Type")
mimeType, _, err := mime.ParseMediaType(contentType)
if err != nil {
return false, fmt.Errorf("parsing Content-Type: %w", err)
}
if mimeType != jsonMimeType {
return false, fmt.Errorf("server should return application/json. Got: %v", contentType)
}
return true, nil
}
func fixCIURL(url string) string {
url = strings.TrimRight(url, "/")
if strings.HasSuffix(url, "/ci") {
url = strings.TrimSuffix(url, "/ci")
}
return url
}
func (n *client) findCertificate(certificate *string, base string, name string) {
if *certificate != "" {
return
}
path := filepath.Join(base, name)
if _, err := os.Stat(path); err == nil {
*certificate = path
}
}
func newClient(requestCredentials requestCredentials) (*client, error) {
url, err := url.Parse(fixCIURL(requestCredentials.GetURL()) + "/api/v4/")
if err != nil {
return nil, err
}
if url.Scheme != "http" && url.Scheme != "https" {
return nil, errors.New("only http or https scheme supported")
}
c := &client{
url: url,
caFile: requestCredentials.GetTLSCAFile(),
certFile: requestCredentials.GetTLSCertFile(),
keyFile: requestCredentials.GetTLSKeyFile(),
requestBackOffs: make(map[string]*backoff.Backoff),
}
c.requester = newRateLimitRequester(&c.Client)
host := strings.Split(url.Host, ":")[0]
if CertificateDirectory != "" {
c.findCertificate(&c.caFile, CertificateDirectory, host+".crt")
c.findCertificate(&c.certFile, CertificateDirectory, host+".auth.crt")
c.findCertificate(&c.keyFile, CertificateDirectory, host+".auth.key")
}
return c, nil
}
package network
import (
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"net/url"
"runtime"
"strconv"
"sync"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
)
const clientError = -100
var apiRequestStatuses = prometheus.NewDesc(
"gitlab_runner_api_request_statuses_total",
"The total number of api requests, partitioned by runner, endpoint and status.",
[]string{"runner", "endpoint", "status"},
nil,
)
type APIEndpoint string
const (
APIEndpointRequestJob APIEndpoint = "request_job"
APIEndpointUpdateJob APIEndpoint = "update_job"
APIEndpointPatchTrace APIEndpoint = "patch_trace"
)
type apiRequestStatusPermutation struct {
runnerID string
endpoint APIEndpoint
status int
}
type APIRequestStatusesMap struct {
internal map[apiRequestStatusPermutation]int
lock sync.RWMutex
}
func (arspm *APIRequestStatusesMap) Append(runnerID string, endpoint APIEndpoint, status int) {
arspm.lock.Lock()
defer arspm.lock.Unlock()
permutation := apiRequestStatusPermutation{runnerID: runnerID, endpoint: endpoint, status: status}
if _, ok := arspm.internal[permutation]; !ok {
arspm.internal[permutation] = 0
}
arspm.internal[permutation]++
}
// Describe implements prometheus.Collector.
func (arspm *APIRequestStatusesMap) Describe(ch chan<- *prometheus.Desc) {
ch <- apiRequestStatuses
}
// Collect implements prometheus.Collector.
func (arspm *APIRequestStatusesMap) Collect(ch chan<- prometheus.Metric) {
arspm.lock.RLock()
defer arspm.lock.RUnlock()
for permutation, count := range arspm.internal {
ch <- prometheus.MustNewConstMetric(
apiRequestStatuses,
prometheus.CounterValue,
float64(count),
permutation.runnerID,
string(permutation.endpoint),
strconv.Itoa(permutation.status),
)
}
}
func NewAPIRequestStatusesMap() *APIRequestStatusesMap {
return &APIRequestStatusesMap{
internal: make(map[apiRequestStatusPermutation]int),
}
}
type GitLabClient struct {
clients map[string]*client
lock sync.Mutex
requestsStatusesMap *APIRequestStatusesMap
}
func (n *GitLabClient) getClient(credentials requestCredentials) (c *client, err error) {
n.lock.Lock()
defer n.lock.Unlock()
if n.clients == nil {
n.clients = make(map[string]*client)
}
key := fmt.Sprintf(
"%s_%s_%s_%s",
credentials.GetURL(),
credentials.GetToken(),
credentials.GetTLSCAFile(),
credentials.GetTLSCertFile(),
)
c = n.clients[key]
if c == nil {
c, err = newClient(credentials)
if err != nil {
return
}
n.clients[key] = c
}
return
}
func (n *GitLabClient) getLastUpdate(credentials requestCredentials) (lu string) {
cli, err := n.getClient(credentials)
if err != nil {
return ""
}
return cli.getLastUpdate()
}
// getFeatures enables features that are properties of networking client
func (n *GitLabClient) getFeatures(features *common.FeaturesInfo) {
features.TraceReset = true
features.TraceChecksum = true
features.TraceSize = true
features.Cancelable = true
}
func (n *GitLabClient) getRunnerVersion(config common.RunnerConfig) common.VersionInfo {
info := common.VersionInfo{
Name: common.NAME,
Version: common.VERSION,
Revision: common.REVISION,
Platform: runtime.GOOS,
Architecture: runtime.GOARCH,
Executor: config.Executor,
Shell: config.Shell,
}
n.getFeatures(&info.Features)
if executorProvider := common.GetExecutorProvider(config.Executor); executorProvider != nil {
_ = executorProvider.GetFeatures(&info.Features)
if info.Shell == "" {
info.Shell = executorProvider.GetDefaultShell()
}
executorProvider.GetConfigInfo(&config, &info.Config)
}
if shell := common.GetShell(info.Shell); shell != nil {
shell.GetFeatures(&info.Features)
}
return info
}
func (n *GitLabClient) doRaw(
ctx context.Context,
credentials requestCredentials,
method, uri string,
request io.Reader,
requestType string,
headers http.Header,
) (res *http.Response, err error) {
c, err := n.getClient(credentials)
if err != nil {
return nil, err
}
return c.do(ctx, uri, method, request, requestType, headers)
}
func (n *GitLabClient) doJSON(
ctx context.Context,
credentials requestCredentials,
method, uri string,
statusCode int,
request interface{},
response interface{},
) (int, string, *http.Response) {
c, err := n.getClient(credentials)
if err != nil {
return clientError, err.Error(), nil
}
return c.doJSON(ctx, uri, method, statusCode, request, response)
}
func (n *GitLabClient) getResponseTLSData(
credentials requestCredentials,
response *http.Response,
) (ResponseTLSData, error) {
c, err := n.getClient(credentials)
if err != nil {
return ResponseTLSData{}, fmt.Errorf("couldn't get client: %w", err)
}
return c.getResponseTLSData(response.TLS)
}
func (n *GitLabClient) RegisterRunner(
runner common.RunnerCredentials,
parameters common.RegisterRunnerParameters,
) *common.RegisterRunnerResponse {
// TODO: pass executor
request := common.RegisterRunnerRequest{
RegisterRunnerParameters: parameters,
Token: runner.Token,
Info: n.getRunnerVersion(common.RunnerConfig{}),
}
var response common.RegisterRunnerResponse
result, statusText, resp := n.doJSON(
context.Background(),
&runner,
http.MethodPost,
"runners",
http.StatusCreated,
&request,
&response,
)
if resp != nil {
defer func() { _ = resp.Body.Close() }()
}
switch result {
case http.StatusCreated:
runner.Log().Println("Registering runner...", "succeeded")
return &response
case http.StatusForbidden:
runner.Log().Errorln("Registering runner...", "forbidden (check registration token)")
return nil
case clientError:
runner.Log().WithField("status", statusText).Errorln("Registering runner...", "error")
return nil
default:
runner.Log().WithField("status", statusText).Errorln("Registering runner...", "failed")
return nil
}
}
func (n *GitLabClient) VerifyRunner(runner common.RunnerCredentials) bool {
request := common.VerifyRunnerRequest{
Token: runner.Token,
}
result, statusText, resp := n.doJSON(
context.Background(),
&runner,
http.MethodPost,
"runners/verify",
http.StatusOK,
&request,
nil,
)
if resp != nil {
defer func() { _ = resp.Body.Close() }()
}
switch result {
case http.StatusOK:
// this is expected due to fact that we ask for non-existing job
runner.Log().Println("Verifying runner...", "is alive")
return true
case http.StatusForbidden:
runner.Log().Errorln("Verifying runner...", "is removed")
return false
case clientError:
runner.Log().WithField("status", statusText).Errorln("Verifying runner...", "error")
return true
default:
runner.Log().WithField("status", statusText).Errorln("Verifying runner...", "failed")
return true
}
}
func (n *GitLabClient) UnregisterRunner(runner common.RunnerCredentials) bool {
request := common.UnregisterRunnerRequest{
Token: runner.Token,
}
result, statusText, resp := n.doJSON(
context.Background(),
&runner,
http.MethodDelete,
"runners",
http.StatusNoContent,
&request,
nil,
)
if resp != nil {
defer func() { _ = resp.Body.Close() }()
}
const baseLogText = "Unregistering runner from GitLab"
switch result {
case http.StatusNoContent:
runner.Log().Println(baseLogText, "succeeded")
return true
case http.StatusForbidden:
runner.Log().Errorln(baseLogText, "forbidden")
return false
case clientError:
runner.Log().WithField("status", statusText).Errorln(baseLogText, "error")
return false
default:
runner.Log().WithField("status", statusText).Errorln(baseLogText, "failed")
return false
}
}
func addTLSData(response *common.JobResponse, tlsData ResponseTLSData) {
if tlsData.CAChain != "" {
response.TLSCAChain = tlsData.CAChain
}
if tlsData.CertFile != "" && tlsData.KeyFile != "" {
data, err := ioutil.ReadFile(tlsData.CertFile)
if err == nil {
response.TLSAuthCert = string(data)
}
data, err = ioutil.ReadFile(tlsData.KeyFile)
if err == nil {
response.TLSAuthKey = string(data)
}
}
}
func (n *GitLabClient) RequestJob(
ctx context.Context,
config common.RunnerConfig,
sessionInfo *common.SessionInfo,
) (*common.JobResponse, bool) {
request := common.JobRequest{
Info: n.getRunnerVersion(config),
Token: config.Token,
LastUpdate: n.getLastUpdate(&config.RunnerCredentials),
Session: sessionInfo,
}
var response common.JobResponse
result, statusText, httpResponse := n.doJSON(
ctx,
&config.RunnerCredentials,
http.MethodPost,
"jobs/request",
http.StatusCreated,
&request,
&response,
)
n.requestsStatusesMap.Append(config.RunnerCredentials.ShortDescription(), APIEndpointRequestJob, result)
switch result {
case http.StatusCreated:
config.Log().WithFields(logrus.Fields{
"job": response.ID,
"repo_url": response.RepoCleanURL(),
}).Println("Checking for jobs...", "received")
tlsData, err := n.getResponseTLSData(&config.RunnerCredentials, httpResponse)
if err != nil {
config.Log().
WithError(err).Errorln("Error on fetching TLS Data from API response...", "error")
}
addTLSData(&response, tlsData)
return &response, true
case http.StatusForbidden:
config.Log().Errorln("Checking for jobs...", "forbidden")
return nil, false
case http.StatusNoContent:
config.Log().Debugln("Checking for jobs...", "nothing")
return nil, true
case clientError:
config.Log().WithField("status", statusText).Errorln("Checking for jobs...", "error")
return nil, false
default:
config.Log().WithField("status", statusText).Warningln("Checking for jobs...", "failed")
return nil, true
}
}
func (n *GitLabClient) UpdateJob(
config common.RunnerConfig,
jobCredentials *common.JobCredentials,
jobInfo common.UpdateJobInfo,
) common.UpdateJobResult {
request := common.UpdateJobRequest{
Info: n.getRunnerVersion(config),
Token: jobCredentials.Token,
State: jobInfo.State,
FailureReason: jobInfo.FailureReason,
Checksum: jobInfo.Output.Checksum, // deprecated
Output: jobInfo.Output,
ExitCode: jobInfo.ExitCode,
}
statusCode, statusText, response := n.doJSON(
context.Background(),
&config.RunnerCredentials,
http.MethodPut,
fmt.Sprintf("jobs/%d", jobInfo.ID),
http.StatusOK,
&request,
nil,
)
n.requestsStatusesMap.Append(config.RunnerCredentials.ShortDescription(), APIEndpointUpdateJob, statusCode)
log := config.Log().WithField("job", jobInfo.ID)
return n.createUpdateJobResult(log, statusCode, statusText, response)
}
func (n *GitLabClient) createUpdateJobResult(
log *logrus.Entry,
statusCode int,
statusText string,
response *http.Response,
) common.UpdateJobResult {
remoteJobStateResponse := NewRemoteJobStateResponse(response, log)
result := common.UpdateJobResult{
NewUpdateInterval: remoteJobStateResponse.RemoteUpdateInterval,
CancelRequested: remoteJobStateResponse.IsCanceled(),
}
log = log.WithFields(logrus.Fields{
"code": statusCode,
"job-status": remoteJobStateResponse.RemoteState,
"update-interval": remoteJobStateResponse.RemoteUpdateInterval,
})
switch {
case remoteJobStateResponse.IsFailed():
log.Warningln("Submitting job to coordinator...", "job failed")
result.State = common.UpdateAbort
case statusCode == http.StatusOK:
log.Debugln("Submitting job to coordinator...", "ok")
result.State = common.UpdateSucceeded
case statusCode == http.StatusAccepted:
log.Debugln("Submitting job to coordinator...", "accepted, but not yet completed")
result.State = common.UpdateAcceptedButNotCompleted
case statusCode == http.StatusPreconditionFailed:
log.Debugln("Submitting job to coordinator...", "trace validation failed")
result.State = common.UpdateTraceValidationFailed
case statusCode == http.StatusNotFound:
log.Warningln("Submitting job to coordinator...", "not found")
result.State = common.UpdateAbort
case statusCode == http.StatusForbidden:
log.WithField("status", statusText).Errorln("Submitting job to coordinator...", "forbidden")
result.State = common.UpdateAbort
case statusCode == clientError:
log.WithField("status", statusText).Errorln("Submitting job to coordinator...", "error")
result.State = common.UpdateAbort
default:
log.WithField("status", statusText).Warningln("Submitting job to coordinator...", "failed")
result.State = common.UpdateFailed
}
return result
}
func (n *GitLabClient) PatchTrace(
config common.RunnerConfig,
jobCredentials *common.JobCredentials,
content []byte,
startOffset int,
) common.PatchTraceResult {
id := jobCredentials.ID
baseLog := config.Log().WithField("job", id)
if len(content) == 0 {
baseLog.Debugln("Appending trace to coordinator...", "skipped due to empty patch")
return common.NewPatchTraceResult(startOffset, common.PatchSucceeded, 0)
}
endOffset := startOffset + len(content)
contentRange := fmt.Sprintf("%d-%d", startOffset, endOffset-1)
headers := make(http.Header)
headers.Set("Content-Range", contentRange)
headers.Set("JOB-TOKEN", jobCredentials.Token)
uri := fmt.Sprintf("jobs/%d/trace", id)
request := bytes.NewReader(content)
response, err := n.doRaw(
context.Background(),
&config.RunnerCredentials,
"PATCH",
uri,
request,
"text/plain",
headers,
)
if err != nil {
config.Log().Errorln("Appending trace to coordinator...", "error", err.Error())
return common.NewPatchTraceResult(startOffset, common.PatchFailed, 0)
}
n.requestsStatusesMap.Append(
config.RunnerCredentials.ShortDescription(),
APIEndpointPatchTrace,
response.StatusCode,
)
defer func() {
_, _ = io.Copy(ioutil.Discard, response.Body)
_ = response.Body.Close()
}()
tracePatchResponse := NewTracePatchResponse(response, baseLog)
log := baseLog.WithFields(logrus.Fields{
"sent-log": contentRange,
"job-log": tracePatchResponse.RemoteRange,
"job-status": tracePatchResponse.RemoteState,
"code": response.StatusCode,
"status": response.Status,
"update-interval": tracePatchResponse.RemoteUpdateInterval,
})
return n.createPatchTraceResult(startOffset, tracePatchResponse, response, endOffset, log)
}
func (n *GitLabClient) createPatchTraceResult(
startOffset int,
tracePatchResponse *TracePatchResponse,
response *http.Response,
endOffset int,
log *logrus.Entry,
) common.PatchTraceResult {
result := common.PatchTraceResult{
SentOffset: startOffset,
NewUpdateInterval: tracePatchResponse.RemoteUpdateInterval,
CancelRequested: tracePatchResponse.IsCanceled(),
}
switch {
case tracePatchResponse.IsFailed():
log.Warningln("Appending trace to coordinator...", "job failed")
result.State = common.PatchAbort
return result
case response.StatusCode == http.StatusAccepted:
log.Debugln("Appending trace to coordinator...", "ok")
result.SentOffset = endOffset
result.State = common.PatchSucceeded
return result
case response.StatusCode == http.StatusNotFound:
log.Warningln("Appending trace to coordinator...", "not-found")
result.State = common.PatchNotFound
return result
case response.StatusCode == http.StatusRequestedRangeNotSatisfiable:
log.Warningln("Appending trace to coordinator...", "range mismatch")
result.SentOffset = tracePatchResponse.NewOffset()
result.State = common.PatchRangeMismatch
return result
case response.StatusCode == clientError:
log.Errorln("Appending trace to coordinator...", "error")
result.State = common.PatchAbort
return result
default:
log.Warningln("Appending trace to coordinator...", "failed")
result.State = common.PatchFailed
return result
}
}
func (n *GitLabClient) createArtifactsForm(reader io.Reader, baseName string) (io.ReadCloser, string) {
pr, pw := io.Pipe()
mpw := multipart.NewWriter(pw)
go func() {
defer func() {
_ = mpw.Close()
_ = pw.Close()
}()
wr, err := mpw.CreateFormFile("file", baseName)
if err != nil {
_ = pw.CloseWithError(err)
return
}
_, err = io.Copy(wr, reader)
if err != nil {
_ = pw.CloseWithError(err)
}
}()
return pr, mpw.FormDataContentType()
}
func uploadRawArtifactsQuery(options common.ArtifactsOptions) url.Values {
q := url.Values{}
if options.ExpireIn != "" {
q.Set("expire_in", options.ExpireIn)
}
if options.Format != "" {
q.Set("artifact_format", string(options.Format))
}
if options.Type != "" {
q.Set("artifact_type", options.Type)
}
return q
}
func (n *GitLabClient) UploadRawArtifacts(
config common.JobCredentials,
reader io.ReadCloser,
options common.ArtifactsOptions,
) common.UploadState {
defer func() {
_ = reader.Close()
}()
pr, contentType := n.createArtifactsForm(reader, options.BaseName)
defer func() {
_ = pr.Close()
}()
query := uploadRawArtifactsQuery(options)
headers := make(http.Header)
headers.Set("JOB-TOKEN", config.Token)
res, err := n.doRaw(
context.Background(),
&config,
http.MethodPost,
fmt.Sprintf("jobs/%d/artifacts?%s", config.ID, query.Encode()),
pr,
contentType,
headers,
)
log := logrus.WithFields(logrus.Fields{
"id": config.ID,
"token": helpers.ShortenToken(config.Token),
})
if res != nil {
log = log.WithField("responseStatus", res.Status)
}
closeWithLogging(log, pr, "pipe reader")
closeWithLogging(log, reader, "archive reader")
messagePrefix := "Uploading artifacts to coordinator..."
if options.Type != "" {
messagePrefix = fmt.Sprintf("Uploading artifacts as %q to coordinator...", options.Type)
}
if err != nil {
log.WithError(err).Errorln(messagePrefix, "error")
return common.UploadFailed
}
defer func() {
_, _ = io.Copy(ioutil.Discard, res.Body)
_ = res.Body.Close()
}()
return n.determineUploadState(res.StatusCode, log, messagePrefix)
}
func closeWithLogging(log logrus.FieldLogger, c io.Closer, name string) {
err := c.Close()
if err != nil {
log.WithError(err).Warningf("Error while closing the %s", name)
}
}
func (n *GitLabClient) determineUploadState(
statusCode int,
log *logrus.Entry,
messagePrefix string,
) common.UploadState {
switch statusCode {
case http.StatusCreated:
log.Println(messagePrefix, "ok")
return common.UploadSucceeded
case http.StatusForbidden:
log.WithField("status", statusCode).Errorln(messagePrefix, "forbidden")
return common.UploadForbidden
case http.StatusRequestEntityTooLarge:
log.WithField("status", statusCode).Errorln(messagePrefix, "too large archive")
return common.UploadTooLarge
case http.StatusServiceUnavailable:
log.WithField("status", statusCode).Errorln(messagePrefix, "service unavailable")
return common.UploadServiceUnavailable
default:
log.WithField("status", statusCode).Warningln(messagePrefix, "failed")
return common.UploadFailed
}
}
func (n *GitLabClient) DownloadArtifacts(
config common.JobCredentials,
artifactsFile io.WriteCloser,
directDownload *bool,
) common.DownloadState {
query := url.Values{}
if directDownload != nil {
query.Set("direct_download", strconv.FormatBool(*directDownload))
}
headers := make(http.Header)
headers.Set("JOB-TOKEN", config.Token)
uri := fmt.Sprintf("jobs/%d/artifacts?%s", config.ID, query.Encode())
res, err := n.doRaw(context.Background(), &config, http.MethodGet, uri, nil, "", headers)
log := logrus.WithFields(logrus.Fields{
"id": config.ID,
"token": helpers.ShortenToken(config.Token),
})
if res != nil {
log = log.WithField("responseStatus", res.Status)
}
if err != nil {
log.Errorln("Downloading artifacts from coordinator...", "error", err.Error())
return common.DownloadFailed
}
defer func() {
_, _ = io.Copy(ioutil.Discard, res.Body)
_ = res.Body.Close()
}()
switch res.StatusCode {
case http.StatusOK:
return n.downloadArtifactFile(log, artifactsFile, res)
case http.StatusForbidden:
log.WithField("status", res.Status).Errorln("Downloading artifacts from coordinator...", "forbidden")
return common.DownloadForbidden
case http.StatusNotFound:
log.Errorln("Downloading artifacts from coordinator...", "not found")
return common.DownloadNotFound
default:
log.WithField("status", res.Status).Warningln("Downloading artifacts from coordinator...", "failed")
return common.DownloadFailed
}
}
func (n *GitLabClient) downloadArtifactFile(
log logrus.FieldLogger,
file io.WriteCloser,
res *http.Response,
) common.DownloadState {
_, err := io.Copy(file, res.Body)
closeWithLogging(log, file, "file writer")
if err != nil {
log.WithError(err).Errorln("Downloading artifacts from coordinator...", "error")
return common.DownloadFailed
}
log.Println("Downloading artifacts from coordinator...", "ok")
return common.DownloadSucceeded
}
func (n *GitLabClient) ProcessJob(
config common.RunnerConfig,
jobCredentials *common.JobCredentials,
) (common.JobTrace, error) {
trace, err := newJobTrace(n, config, jobCredentials)
if err != nil {
return nil, err
}
trace.start()
return trace, nil
}
func NewGitLabClientWithRequestStatusesMap(rsMap *APIRequestStatusesMap) *GitLabClient {
return &GitLabClient{
requestsStatusesMap: rsMap,
}
}
func NewGitLabClient() *GitLabClient {
return NewGitLabClientWithRequestStatusesMap(NewAPIRequestStatusesMap())
}
package network
import (
"net/http"
"strconv"
"strings"
"github.com/sirupsen/logrus"
)
const (
rangeHeader = "Range"
)
type TracePatchResponse struct {
*RemoteJobStateResponse
RemoteRange string
}
func (p *TracePatchResponse) NewOffset() int {
remoteRangeParts := strings.Split(p.RemoteRange, "-")
if len(remoteRangeParts) == 2 {
newOffset, _ := strconv.Atoi(remoteRangeParts[1])
return newOffset
}
return 0
}
func NewTracePatchResponse(response *http.Response, logger logrus.FieldLogger) *TracePatchResponse {
result := &TracePatchResponse{
RemoteJobStateResponse: NewRemoteJobStateResponse(response, logger),
}
if response != nil {
result.RemoteRange = response.Header.Get(rangeHeader)
}
return result
}
package network
import (
"errors"
"fmt"
"net/http"
"time"
"github.com/sirupsen/logrus"
)
// NOTE: The functionality of the rate limiting below as well as the constant values
// are documented in `docs/configuration/proxy.md#handling-rate-limited-requests`
const (
// RateLimit-ResetTime: Wed, 21 Oct 2015 07:28:00 GMT
rateLimitResetTimeHeader = "RateLimit-ResetTime"
// The fallback is used if the reset header's value is present but cannot be parsed
defaultRateLimitFallbackDelay = time.Minute
defaultRateLimitRetriesCount = 5
)
var (
errRateLimitGaveUp = errors.New("gave up due to rate limit")
)
type rateLimitRequester struct {
client requester
fallbackDelay time.Duration
retriesCount int
}
func newRateLimitRequester(client requester) *rateLimitRequester {
return &rateLimitRequester{
client: client,
fallbackDelay: defaultRateLimitFallbackDelay,
retriesCount: defaultRateLimitRetriesCount,
}
}
func (r *rateLimitRequester) Do(req *http.Request) (*http.Response, error) {
logger := logrus.
WithFields(logrus.Fields{
"context": "ratelimit-requester-gitlab-request",
"url": req.URL.String(),
"method": req.Method,
})
// Worst case would be the configured timeout from reverse proxy * retriesCount
for i := 0; i < r.retriesCount; i++ {
res, rateLimitDuration, err := r.do(req, logger)
if rateLimitDuration == nil {
return res, err
}
logger.
WithField("duration", *rateLimitDuration).
Infoln("Sleeping due to rate limit")
// In some rare cases where the network is slow or the machine hosting
// the runner is resource constrained by the time we get the header
// it might be in the past, but that's ok since sleep will return immediately
time.Sleep(*rateLimitDuration)
}
return nil, errRateLimitGaveUp
}
// If this method returns a non-nil duration this means that we got a rate limited response
// and the called should sleep for the duration. If the duration is nil, return the response and the error
// meaning that we got a non rate limited response
func (r *rateLimitRequester) do(req *http.Request, logger *logrus.Entry) (*http.Response, *time.Duration, error) {
res, err := r.client.Do(req)
if err != nil {
return nil, nil, fmt.Errorf("couldn't execute %s against %s: %w", req.Method, req.URL, err)
}
// The request passed and we got some non rate limited response
if res.StatusCode != http.StatusTooManyRequests {
return res, nil, nil
}
rateLimitResetTimeValue := res.Header.Get(rateLimitResetTimeHeader)
if rateLimitResetTimeValue == "" {
// if we get a 429 but don't have a rate limit reset header we just return the response
// since we can't know how much to wait for the rate limit to reset
return res, nil, nil
}
resetTime, err := time.Parse(time.RFC1123, rateLimitResetTimeValue)
if err != nil {
// If we can't parse the rate limit reset header there's something wrong with it
// we shouldn't fail, to avoid a case where a misconfiguration in the reverse proxy can cause
// all runners to stop working. Wait for the configured fallback instead
logger.
WithError(err).
WithFields(logrus.Fields{
"header": rateLimitResetTimeHeader,
"headerValue": rateLimitResetTimeValue,
}).
Warnln("Couldn't parse rate limit header, falling back")
return res, &r.fallbackDelay, nil
}
resetDuration := time.Until(resetTime)
return res, &resetDuration, nil
}
package network
import (
"net/http"
"strconv"
"time"
"github.com/sirupsen/logrus"
)
const (
updateIntervalHeader = "X-GitLab-Trace-Update-Interval"
remoteStateHeader = "Job-Status"
statusCanceling = "canceling"
statusCanceled = "canceled"
statusFailed = "failed"
)
type RemoteJobStateResponse struct {
StatusCode int
RemoteState string
RemoteUpdateInterval time.Duration
}
func (r *RemoteJobStateResponse) IsFailed() bool {
if r.RemoteState == statusCanceled || r.RemoteState == statusFailed {
return true
}
if r.StatusCode == http.StatusForbidden {
return true
}
return false
}
func (r *RemoteJobStateResponse) IsCanceled() bool {
return r.RemoteState == statusCanceling
}
func NewRemoteJobStateResponse(response *http.Response, logger logrus.FieldLogger) *RemoteJobStateResponse {
if response == nil {
return &RemoteJobStateResponse{}
}
result := &RemoteJobStateResponse{
StatusCode: response.StatusCode,
RemoteState: response.Header.Get(remoteStateHeader),
}
if updateIntervalRaw := response.Header.Get(updateIntervalHeader); updateIntervalRaw != "" {
if updateInterval, err := strconv.Atoi(updateIntervalRaw); err == nil {
result.RemoteUpdateInterval = time.Duration(updateInterval) * time.Second
} else {
logger.WithError(err).
WithField("header-value", updateIntervalRaw).
Warningf("Failed to parse %q header", updateIntervalHeader)
}
}
return result
}
package network
import (
"context"
"sync"
"time"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/trace"
)
type clientJobTrace struct {
client common.Network
config common.RunnerConfig
jobCredentials *common.JobCredentials
id int
cancelFunc context.CancelFunc
abortFunc context.CancelFunc
buffer *trace.Buffer
lock sync.RWMutex
state common.JobState
failureReason common.JobFailureReason
finished chan bool
sentTrace int
sentTime time.Time
updateInterval time.Duration
forceSendInterval time.Duration
maxTracePatchSize int
failuresCollector common.FailuresCollector
exitCode int
}
func (c *clientJobTrace) Success() {
c.complete(nil, common.JobFailureData{})
}
func (c *clientJobTrace) complete(err error, failureData common.JobFailureData) {
c.lock.Lock()
if c.state != common.Running {
c.lock.Unlock()
return
}
if err == nil {
c.state = common.Success
} else {
c.setFailure(failureData)
}
c.lock.Unlock()
c.finish()
}
func (c *clientJobTrace) Fail(err error, failureData common.JobFailureData) {
c.complete(err, failureData)
}
func (c *clientJobTrace) Write(data []byte) (n int, err error) {
return c.buffer.Write(data)
}
func (c *clientJobTrace) SetMasked(masked []string) {
c.buffer.SetMasked(masked)
}
func (c *clientJobTrace) checksum() string {
return c.buffer.Checksum()
}
func (c *clientJobTrace) bytesize() int {
return c.buffer.Size()
}
// SetCancelFunc sets the function to be called by Cancel(). The function
// provided here should cancel the execution of any stages that are not
// absolutely required, whilst allowing for stages such as `after_script` to
// proceed.
func (c *clientJobTrace) SetCancelFunc(cancelFunc context.CancelFunc) {
c.lock.Lock()
defer c.lock.Unlock()
c.cancelFunc = cancelFunc
}
// Cancel consumes the function set by SetCancelFunc.
func (c *clientJobTrace) Cancel() bool {
c.lock.RLock()
cancelFunc := c.cancelFunc
c.lock.RUnlock()
if cancelFunc == nil {
return false
}
c.SetCancelFunc(nil)
cancelFunc()
return true
}
// SetAbortFunc sets the function to be called by Abort(). The function
// provided here should abort the execution of all stages.
func (c *clientJobTrace) SetAbortFunc(cancelFunc context.CancelFunc) {
c.lock.Lock()
defer c.lock.Unlock()
c.abortFunc = cancelFunc
}
// Abort consumes function set by SetAbortFunc
// The abort always have much higher importance than Cancel
// as abort interrupts the execution, thus cancel is never
// called after the Abort
func (c *clientJobTrace) Abort() bool {
c.lock.RLock()
abortFunc := c.abortFunc
c.lock.RUnlock()
if abortFunc == nil {
return false
}
c.SetCancelFunc(nil)
c.SetAbortFunc(nil)
abortFunc()
return true
}
func (c *clientJobTrace) SetFailuresCollector(fc common.FailuresCollector) {
c.failuresCollector = fc
}
func (c *clientJobTrace) IsStdout() bool {
return false
}
func (c *clientJobTrace) setFailure(data common.JobFailureData) {
c.state = common.Failed
c.failureReason = data.Reason
c.exitCode = data.ExitCode
if c.failuresCollector != nil {
c.failuresCollector.RecordFailure(data.Reason, c.config.ShortDescription())
}
}
func (c *clientJobTrace) start() {
c.finished = make(chan bool)
c.state = common.Running
c.setupLogLimit()
go c.watch()
}
func (c *clientJobTrace) ensureAllTraceSent() {
for c.anyTraceToSend() {
switch c.sendPatch().State {
case common.PatchSucceeded:
// we continue sending till we succeed
continue
case common.PatchAbort:
return
case common.PatchNotFound:
return
case common.PatchRangeMismatch:
time.Sleep(c.getUpdateInterval())
case common.PatchFailed:
time.Sleep(c.getUpdateInterval())
}
}
}
func (c *clientJobTrace) finalUpdate() {
// On final-update we want the Runner to fallback
// to default interval and make Rails to override it
c.setUpdateInterval(common.DefaultUpdateInterval)
for {
// Before sending update to ensure that trace is sent
// as `sendUpdate()` can force Runner to rewind trace
c.ensureAllTraceSent()
switch c.sendUpdate() {
case common.UpdateSucceeded:
return
case common.UpdateAbort:
return
case common.UpdateNotFound:
return
case common.UpdateAcceptedButNotCompleted:
time.Sleep(c.getUpdateInterval())
case common.UpdateTraceValidationFailed:
time.Sleep(c.getUpdateInterval())
case common.UpdateFailed:
time.Sleep(c.getUpdateInterval())
}
}
}
func (c *clientJobTrace) finish() {
c.buffer.Finish()
c.finished <- true
c.finalUpdate()
c.buffer.Close()
}
// incrementalUpdate returns a flag if jobs is supposed
// to be running, or whether it should be finished
func (c *clientJobTrace) incrementalUpdate() bool {
patchResult := c.sendPatch()
if patchResult.CancelRequested {
c.Cancel()
}
switch patchResult.State {
case common.PatchSucceeded:
// We try to additionally touch job to check
// it might be required if no content was send
// for longer period of time.
// This is needed to discover if it should be aborted
touchResult := c.touchJob()
if touchResult.CancelRequested {
c.Cancel()
}
if touchResult.State == common.UpdateAbort {
c.Abort()
return false
}
case common.PatchAbort:
c.Abort()
return false
}
return true
}
func (c *clientJobTrace) anyTraceToSend() bool {
c.lock.RLock()
defer c.lock.RUnlock()
return c.buffer.Size() != c.sentTrace
}
func (c *clientJobTrace) sendPatch() common.PatchTraceResult {
c.lock.RLock()
content, err := c.buffer.Bytes(c.sentTrace, c.maxTracePatchSize)
sentTrace := c.sentTrace
c.lock.RUnlock()
if err != nil {
return common.PatchTraceResult{State: common.PatchFailed}
}
if len(content) == 0 {
return common.PatchTraceResult{State: common.PatchSucceeded}
}
result := c.client.PatchTrace(c.config, c.jobCredentials, content, sentTrace)
c.setUpdateInterval(result.NewUpdateInterval)
if result.State == common.PatchSucceeded || result.State == common.PatchRangeMismatch {
c.lock.Lock()
c.sentTime = time.Now()
c.sentTrace = result.SentOffset
c.lock.Unlock()
}
return result
}
func (c *clientJobTrace) setUpdateInterval(newUpdateInterval time.Duration) {
if newUpdateInterval <= 0 {
return
}
c.lock.Lock()
defer c.lock.Unlock()
c.updateInterval = newUpdateInterval
// Let's hope that this never happens,
// but if server behaves bogus do not have too long interval
if c.updateInterval > common.MaxUpdateInterval {
c.updateInterval = common.MaxUpdateInterval
}
}
// Update Coordinator that the job is still running.
func (c *clientJobTrace) touchJob() common.UpdateJobResult {
c.lock.RLock()
shouldRefresh := time.Since(c.sentTime) > c.forceSendInterval
c.lock.RUnlock()
if !shouldRefresh {
return common.UpdateJobResult{State: common.UpdateSucceeded}
}
jobInfo := common.UpdateJobInfo{
ID: c.id,
State: common.Running,
Output: common.JobTraceOutput{
Checksum: c.checksum(),
Bytesize: c.bytesize(),
},
}
result := c.client.UpdateJob(c.config, c.jobCredentials, jobInfo)
c.setUpdateInterval(result.NewUpdateInterval)
if result.State == common.UpdateSucceeded {
c.lock.Lock()
c.sentTime = time.Now()
c.lock.Unlock()
}
return result
}
func (c *clientJobTrace) sendUpdate() common.UpdateState {
c.lock.RLock()
state := c.state
c.lock.RUnlock()
jobInfo := common.UpdateJobInfo{
ID: c.id,
State: state,
FailureReason: c.failureReason,
Output: common.JobTraceOutput{
Checksum: c.checksum(),
Bytesize: c.bytesize(),
},
ExitCode: c.exitCode,
}
result := c.client.UpdateJob(c.config, c.jobCredentials, jobInfo)
c.setUpdateInterval(result.NewUpdateInterval)
if result.State == common.UpdateSucceeded {
c.lock.Lock()
c.sentTime = time.Now()
c.lock.Unlock()
} else if result.State == common.UpdateTraceValidationFailed {
c.lock.Lock()
c.sentTime = time.Now()
c.sentTrace = 0
c.lock.Unlock()
}
return result.State
}
func (c *clientJobTrace) watch() {
for {
select {
case <-time.After(c.getUpdateInterval()):
if !c.incrementalUpdate() {
// job is no longer running, wait for finish
<-c.finished
return
}
case <-c.finished:
return
}
}
}
func (c *clientJobTrace) getUpdateInterval() time.Duration {
c.lock.RLock()
defer c.lock.RUnlock()
return c.updateInterval
}
func (c *clientJobTrace) setupLogLimit() {
bytesLimit := c.config.OutputLimit * 1024 // convert to bytes
if bytesLimit == 0 {
bytesLimit = common.DefaultTraceOutputLimit
}
c.buffer.SetLimit(bytesLimit)
}
func newJobTrace(
client common.Network,
config common.RunnerConfig,
jobCredentials *common.JobCredentials,
) (*clientJobTrace, error) {
buffer, err := trace.New()
if err != nil {
return nil, err
}
return &clientJobTrace{
client: client,
config: config,
buffer: buffer,
jobCredentials: jobCredentials,
id: jobCredentials.ID,
maxTracePatchSize: common.DefaultTracePatchLimit,
updateInterval: common.DefaultUpdateInterval,
forceSendInterval: common.TraceForceSendInterval,
}, nil
}
package referees
import (
"bytes"
"context"
"encoding/json"
"fmt"
"reflect"
"strings"
"time"
"github.com/prometheus/client_golang/api"
prometheusV1 "github.com/prometheus/client_golang/api/prometheus/v1"
"github.com/prometheus/common/model"
"github.com/sirupsen/logrus"
)
type MetricsReferee struct {
prometheusAPI prometheusV1.API
queries []string
queryInterval time.Duration
selector string
logger logrus.FieldLogger
}
//nolint:lll
type MetricsRefereeConfig struct {
PrometheusAddress string `toml:"prometheus_address,omitempty" json:"prometheus_address" description:"A host:port to a prometheus metrics server"`
QueryInterval int `toml:"query_interval,omitempty" json:"query_interval" description:"Query interval (in seconds)"`
Queries []string `toml:"queries" json:"queries" description:"A list of metrics to query (in PromQL)"`
}
type MetricsExecutor interface {
GetMetricsSelector() string
}
func (mr *MetricsReferee) ArtifactBaseName() string {
return "metrics_referee.json"
}
func (mr *MetricsReferee) ArtifactType() string {
return "metrics_referee"
}
func (mr *MetricsReferee) ArtifactFormat() string {
return "gzip"
}
func (mr *MetricsReferee) Execute(ctx context.Context, startTime, endTime time.Time) (*bytes.Reader, error) {
// specify the range used for the PromQL query
queryRange := prometheusV1.Range{
Start: startTime.UTC(),
End: endTime.UTC(),
Step: mr.queryInterval,
}
metrics := make(map[string][]model.SamplePair)
// use config file to pull metrics from prometheus range queries
for _, metricQuery := range mr.queries {
// break up query into name:query
components := strings.Split(metricQuery, ":")
if len(components) != 2 {
err := fmt.Errorf("%q not in name:query format in metric queries", metricQuery)
mr.logger.WithError(err).Error("Failed to parse metrics query")
return nil, err
}
name := components[0]
query := components[1]
result := mr.queryMetrics(ctx, query, queryRange)
if result == nil {
continue
}
metrics[name] = result
}
// convert metrics sample pairs to JSON
output, err := json.Marshal(metrics)
if err != nil {
return nil, err
}
return bytes.NewReader(output), nil
}
func (mr *MetricsReferee) queryMetrics(
ctx context.Context,
query string,
queryRange prometheusV1.Range,
) []model.SamplePair {
interval := fmt.Sprintf("%.0fs", mr.queryInterval.Seconds())
query = strings.ReplaceAll(query, "{selector}", mr.selector)
query = strings.ReplaceAll(query, "{interval}", interval)
queryLogger := mr.logger.WithFields(logrus.Fields{
"query": query,
"start": queryRange.Start,
"end": queryRange.End,
})
queryLogger.Debug("Sending request to Prometheus API")
// execute query over range
result, _, err := mr.prometheusAPI.QueryRange(ctx, query, queryRange)
if err != nil {
queryLogger.WithError(err).Error("Failed to range query Prometheus")
return nil
}
if result == nil {
queryLogger.Error("Received nil range query result")
return nil
}
// ensure matrix result
matrix, ok := result.(model.Matrix)
if !ok {
queryLogger.
WithField("result-type", reflect.TypeOf(result)).
Info("Failed to type assert result into model.Matrix")
return nil
}
// no results for range query
if matrix.Len() == 0 {
return nil
}
// save first result set values at metric
return matrix[0].Values
}
func newMetricsReferee(executor interface{}, config *Config, log logrus.FieldLogger) Referee {
logger := log.WithField("referee", "metrics")
if config.Metrics == nil {
return nil
}
// see if provider supports metrics refereeing
refereed, ok := executor.(MetricsExecutor)
if !ok {
logger.Info("executor not supported")
return nil
}
// create prometheus client from server address in config
clientConfig := api.Config{Address: config.Metrics.PrometheusAddress}
prometheusClient, err := api.NewClient(clientConfig)
if err != nil {
logger.WithError(err).Error("failed to create prometheus client")
return nil
}
prometheusAPI := prometheusV1.NewAPI(prometheusClient)
return &MetricsReferee{
prometheusAPI: prometheusAPI,
queryInterval: time.Duration(config.Metrics.QueryInterval) * time.Second,
queries: config.Metrics.Queries,
selector: refereed.GetMetricsSelector(),
logger: logger,
}
}
package referees
import (
"bytes"
"context"
"time"
"github.com/sirupsen/logrus"
)
type Referee interface {
Execute(
ctx context.Context,
startTime time.Time,
endTime time.Time,
) (*bytes.Reader, error)
ArtifactBaseName() string
ArtifactType() string
ArtifactFormat() string
}
type refereeFactory func(executor interface{}, config *Config, log logrus.FieldLogger) Referee
type Config struct {
Metrics *MetricsRefereeConfig `toml:"metrics,omitempty" json:"metrics" namespace:"metrics"`
}
var refereeFactories = []refereeFactory{
newMetricsReferee,
}
func CreateReferees(executor interface{}, config *Config, log logrus.FieldLogger) []Referee {
if config == nil {
log.Debug("No referees configured")
return nil
}
var referees []Referee
for _, factory := range refereeFactories {
referee := factory(executor, config, log)
if referee != nil {
referees = append(referees, referee)
}
}
return referees
}
package proxy
import (
"errors"
"net/http"
"strconv"
)
type Pool map[string]*Proxy
type Pooler interface {
Pool() Pool
}
type Proxy struct {
Settings *Settings
ConnectionHandler Requester
}
type Settings struct {
ServiceName string
Ports []Port
}
type Port struct {
Number int
Protocol string
Name string
}
type Requester interface {
ProxyRequest(w http.ResponseWriter, r *http.Request, requestedURI, port string, settings *Settings)
}
func NewPool() Pool {
return Pool{}
}
func NewProxySettings(serviceName string, ports []Port) *Settings {
return &Settings{
ServiceName: serviceName,
Ports: ports,
}
}
// PortByNameOrNumber accepts both a port number or a port name.
// It will try to convert the method into an integer and then
// search if there is any port number with that value or any
// port name by the param value.
func (p *Settings) PortByNameOrNumber(portNameOrNumber string) (Port, error) {
intPort, _ := strconv.Atoi(portNameOrNumber)
for _, port := range p.Ports {
if port.Number == intPort || port.Name == portNameOrNumber {
return port, nil
}
}
return Port{}, errors.New("invalid port")
}
func (p *Port) Scheme() (string, error) {
if p.Protocol == "http" || p.Protocol == "https" {
return p.Protocol, nil
}
return "", errors.New("invalid port scheme")
}
// WebsocketProtocolFor returns the proper Websocket protocol
// based on the HTTP protocol
func WebsocketProtocolFor(httpProtocol string) string {
if httpProtocol == "https" {
return "wss"
}
return "ws"
}
package session
import (
"crypto/tls"
"errors"
"net"
"net/http"
"net/url"
"strings"
"time"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/helpers/certificate"
)
type sessionFinderFn func(url string) *Session
type Server struct {
config ServerConfig
log *logrus.Entry
tlsListener net.Listener
sessionFinder sessionFinderFn
httpServer *http.Server
CertificatePublicKey []byte
AdvertiseAddress string
}
type ServerConfig struct {
AdvertiseAddress string
ListenAddress string
ShutdownTimeout time.Duration
}
func NewServer(
config ServerConfig,
logger *logrus.Entry,
certGen certificate.Generator,
sessionFinder sessionFinderFn,
) (*Server, error) {
if logger == nil {
logger = logrus.NewEntry(logrus.StandardLogger())
}
server := Server{
config: config,
log: logger,
sessionFinder: sessionFinder,
httpServer: &http.Server{},
}
host, err := server.getPublicHost()
if err != nil {
return nil, err
}
cert, publicKey, err := certGen.Generate(host)
if err != nil {
return nil, err
}
tlsConfig := tls.Config{
Certificates: []tls.Certificate{cert},
MinVersion: tls.VersionTLS12,
}
// We separate out the listener creation here so that we can return an error
// if the provided address is invalid or there is some other listener error.
listener, err := net.Listen("tcp", server.config.ListenAddress)
if err != nil {
return nil, err
}
server.tlsListener = tls.NewListener(listener, &tlsConfig)
err = server.setAdvertiseAddress()
if err != nil {
return nil, err
}
server.CertificatePublicKey = publicKey
server.httpServer.Handler = http.HandlerFunc(server.handleSessionRequest)
return &server, nil
}
func (s *Server) getPublicHost() (string, error) {
for _, address := range []string{s.config.AdvertiseAddress, s.config.ListenAddress} {
if address == "" {
continue
}
host, _, err := net.SplitHostPort(address)
if err != nil {
s.log.
WithField("address", address).
WithError(err).
Warn("Failed to parse session address")
}
if host == "" {
continue
}
return host, nil
}
return "", errors.New("no valid address provided")
}
func (s *Server) setAdvertiseAddress() error {
s.AdvertiseAddress = s.config.AdvertiseAddress
if s.config.AdvertiseAddress == "" {
s.AdvertiseAddress = s.config.ListenAddress
}
if strings.HasPrefix(s.AdvertiseAddress, "https://") ||
strings.HasPrefix(s.AdvertiseAddress, "http://") {
return errors.New("url not valid, scheme defined")
}
s.AdvertiseAddress = "https://" + s.AdvertiseAddress
_, err := url.ParseRequestURI(s.AdvertiseAddress)
return err
}
func (s *Server) handleSessionRequest(w http.ResponseWriter, r *http.Request) {
logger := s.log.WithField("uri", r.RequestURI)
logger.Debug("Processing session request")
session := s.sessionFinder(r.RequestURI)
if session == nil || session.Mux() == nil {
logger.Error("Mux handler not found")
http.NotFound(w, r)
return
}
session.Mux().ServeHTTP(w, r)
}
func (s *Server) Start() error {
if s.httpServer == nil {
return errors.New("http server not set")
}
err := s.httpServer.Serve(s.tlsListener)
// ErrServerClosed is a legitimate error that should not cause failure
if err == http.ErrServerClosed {
return nil
}
return err
}
func (s *Server) Close() {
if s.httpServer != nil {
_ = s.httpServer.Close()
}
}
package session
import (
"net/http"
"reflect"
"sync"
"github.com/gorilla/mux"
"github.com/gorilla/websocket"
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
"gitlab.com/gitlab-org/gitlab-runner/session/proxy"
"gitlab.com/gitlab-org/gitlab-runner/session/terminal"
)
type connectionInUseError struct{}
func (connectionInUseError) Error() string {
return "Connection already in use"
}
type Session struct {
Endpoint string
Token string
mux *mux.Router
interactiveTerminal terminal.InteractiveTerminal
terminalConn terminal.Conn
proxyPool proxy.Pool
// Signal when client disconnects from terminal.
DisconnectCh chan error
// Signal when terminal session timeout.
TimeoutCh chan error
log *logrus.Entry
lock sync.Mutex
}
func NewSession(logger *logrus.Entry) (*Session, error) {
endpoint, token, err := generateEndpoint()
if err != nil {
return nil, err
}
if logger == nil {
logger = logrus.NewEntry(logrus.StandardLogger())
}
logger = logger.WithField("uri", endpoint)
sess := &Session{
Endpoint: endpoint,
Token: token,
DisconnectCh: make(chan error),
TimeoutCh: make(chan error),
log: logger,
}
sess.setMux()
return sess, nil
}
func generateEndpoint() (string, string, error) {
sessionUUID, err := helpers.GenerateRandomUUID(32)
if err != nil {
return "", "", err
}
token, err := generateToken()
if err != nil {
return "", "", err
}
return "/session/" + sessionUUID, token, nil
}
func generateToken() (string, error) {
token, err := helpers.GenerateRandomUUID(32)
if err != nil {
return "", err
}
return token, nil
}
func (s *Session) withAuthorization(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
logger := s.log.WithField("uri", r.RequestURI)
logger.Debug("Endpoint session request")
if s.Token != r.Header.Get("Authorization") {
logger.Error("Authorization header is not valid")
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return
}
next.ServeHTTP(w, r)
})
}
func (s *Session) setMux() {
s.lock.Lock()
defer s.lock.Unlock()
s.mux = mux.NewRouter()
s.mux.Handle(
s.Endpoint+"/proxy/{resource}/{port}/{requestedUri:.*}",
s.withAuthorization(http.HandlerFunc(s.proxyHandler)),
)
s.mux.Handle(s.Endpoint+"/exec", s.withAuthorization(http.HandlerFunc(s.execHandler)))
}
func (s *Session) proxyHandler(w http.ResponseWriter, r *http.Request) {
logger := s.log.WithField("uri", r.RequestURI)
logger.Debug("Proxy session request")
params := mux.Vars(r)
serviceName := params["resource"]
serviceProxy := s.proxyPool[serviceName]
if serviceProxy == nil {
logger.Warn("Proxy not found")
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
if serviceProxy.ConnectionHandler == nil {
logger.Warn("Proxy connection handler is not defined")
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
serviceProxy.ConnectionHandler.ProxyRequest(w, r, params["requestedUri"], params["port"], serviceProxy.Settings)
}
func (s *Session) execHandler(w http.ResponseWriter, r *http.Request) {
logger := s.log.WithField("uri", r.RequestURI)
logger.Debug("Exec terminal session request")
if !s.terminalAvailable() {
logger.Error("Interactive terminal not set")
http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
return
}
if !websocket.IsWebSocketUpgrade(r) {
logger.Error("Request is not a web socket connection")
http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
return
}
terminalConn, err := s.newTerminalConn()
if _, ok := err.(connectionInUseError); ok {
logger.Warn("Terminal already connected, revoking connection")
http.Error(w, http.StatusText(http.StatusLocked), http.StatusLocked)
return
}
if err != nil {
logger.WithError(err).Error("Failed to connect to terminal")
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
defer s.closeTerminalConn(terminalConn)
logger.Debugln("Starting terminal session")
terminalConn.Start(w, r, s.TimeoutCh, s.DisconnectCh)
}
func (s *Session) terminalAvailable() bool {
s.lock.Lock()
defer s.lock.Unlock()
return s.interactiveTerminal != nil
}
func (s *Session) newTerminalConn() (terminal.Conn, error) {
s.lock.Lock()
defer s.lock.Unlock()
if s.terminalConn != nil {
return nil, connectionInUseError{}
}
conn, err := s.interactiveTerminal.Connect()
if err != nil {
return nil, err
}
s.terminalConn = conn
return conn, nil
}
func (s *Session) closeTerminalConn(conn terminal.Conn) {
s.lock.Lock()
defer s.lock.Unlock()
err := conn.Close()
if err != nil {
s.log.WithError(err).Warn("Failed to close terminal connection")
}
if reflect.ValueOf(s.terminalConn) == reflect.ValueOf(conn) {
s.log.Warningln("Closed active terminal connection")
s.terminalConn = nil
}
}
func (s *Session) SetInteractiveTerminal(interactiveTerminal terminal.InteractiveTerminal) {
s.lock.Lock()
defer s.lock.Unlock()
s.interactiveTerminal = interactiveTerminal
}
func (s *Session) SetProxyPool(pooler proxy.Pooler) {
s.lock.Lock()
defer s.lock.Unlock()
s.proxyPool = pooler.Pool()
}
func (s *Session) Mux() *mux.Router {
return s.mux
}
func (s *Session) Connected() bool {
s.lock.Lock()
defer s.lock.Unlock()
return s.terminalConn != nil
}
func (s *Session) Kill() error {
s.lock.Lock()
defer s.lock.Unlock()
if s.terminalConn == nil {
return nil
}
err := s.terminalConn.Close()
s.terminalConn = nil
return err
}
package terminal
import (
"errors"
"net/http"
)
type InteractiveTerminal interface {
Connect() (Conn, error)
}
type Conn interface {
Start(w http.ResponseWriter, r *http.Request, timeoutCh, disconnectCh chan error)
Close() error
}
func ProxyTerminal(timeoutCh, disconnectCh, proxyStopCh chan error, proxyFunc func()) {
disconnected := make(chan bool, 1)
// terminal exit handler
go func() {
// wait for either session timeout or disconnection from the client
select {
case err := <-timeoutCh:
proxyStopCh <- err
case <-disconnected:
// forward the disconnection event if there is any waiting receiver
nonBlockingSend(
disconnectCh,
errors.New("finished proxying (client disconnected?)"),
)
}
}()
proxyFunc()
disconnected <- true
}
func nonBlockingSend(ch chan error, err error) {
select {
case ch <- err:
default:
}
}
package shells
import (
"errors"
"fmt"
"net/url"
"path"
"path/filepath"
"strconv"
"strings"
"gitlab.com/gitlab-org/gitlab-runner/cache"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
"gitlab.com/gitlab-org/gitlab-runner/helpers/tls"
)
type AbstractShell struct {
}
func (b *AbstractShell) GetFeatures(features *common.FeaturesInfo) {
features.Artifacts = true
features.UploadMultipleArtifacts = true
features.UploadRawArtifacts = true
features.Cache = true
features.Refspecs = true
features.Masking = true
features.RawVariables = true
features.ArtifactsExclude = true
features.MultiBuildSteps = true
features.VaultSecrets = true
features.ReturnExitCode = true
}
func (b *AbstractShell) writeCdBuildDir(w ShellWriter, info common.ShellScriptInfo) {
w.Cd(info.Build.FullProjectDir())
}
func (b *AbstractShell) cacheFile(build *common.Build, userKey string) (key, file string) {
if build.CacheDir == "" {
return
}
// Deduce cache key
key = path.Join(build.JobInfo.Name, build.GitInfo.Ref)
if userKey != "" {
key = build.GetAllVariables().ExpandValue(userKey)
}
// Ignore cache without the key
if key == "" {
return
}
file = path.Join(build.CacheDir, key, "cache.zip")
if build.IsFeatureFlagOn(featureflags.UsePowershellPathResolver) {
return key, file
}
file, err := filepath.Rel(build.BuildDir, file)
if err != nil {
return "", ""
}
return
}
func (b *AbstractShell) guardRunnerCommand(w ShellWriter, runnerCommand string, action string, f func()) {
if runnerCommand == "" {
w.Warningf("%s is not supported by this executor.", action)
return
}
w.IfCmd(runnerCommand, "--version")
f()
w.Else()
w.Warningf("Missing %s. %s is disabled.", runnerCommand, action)
w.EndIf()
}
func (b *AbstractShell) cacheExtractor(w ShellWriter, info common.ShellScriptInfo) error {
skipRestoreCache := true
for _, cacheOptions := range info.Build.Cache {
// Create list of files to extract
var archiverArgs []string
for _, path := range cacheOptions.Paths {
archiverArgs = append(archiverArgs, "--path", path)
}
if cacheOptions.Untracked {
archiverArgs = append(archiverArgs, "--untracked")
}
// Skip restoring cache if no cache is defined
if len(archiverArgs) < 1 {
continue
}
skipRestoreCache = false
// Skip extraction if no cache is defined
cacheKey, cacheFile := b.cacheFile(info.Build, cacheOptions.Key)
if cacheKey == "" {
w.Noticef("Skipping cache extraction due to empty cache key")
continue
}
if ok, err := cacheOptions.CheckPolicy(common.CachePolicyPull); err != nil {
return fmt.Errorf("%w for %s", err, cacheKey)
} else if !ok {
w.Noticef("Not downloading cache %s due to policy", cacheKey)
continue
}
b.extractCacheOrFallbackCacheWrapper(w, info, cacheFile, cacheKey)
}
if skipRestoreCache {
return common.ErrSkipBuildStage
}
return nil
}
func (b *AbstractShell) extractCacheOrFallbackCacheWrapper(
w ShellWriter,
info common.ShellScriptInfo,
cacheFile string,
cacheKey string,
) {
cacheFallbackKey := info.Build.GetAllVariables().Get("CACHE_FALLBACK_KEY")
// Execute cache-extractor command. Failure is not fatal.
b.guardRunnerCommand(w, info.RunnerCommand, "Extracting cache", func() {
b.addExtractCacheCommand(w, info, cacheFile, cacheKey, cacheFallbackKey)
})
}
func (b *AbstractShell) addExtractCacheCommand(
w ShellWriter,
info common.ShellScriptInfo,
cacheFile string,
cacheKey string,
cacheFallbackKey string,
) {
args := []string{
"cache-extractor",
"--file", cacheFile,
"--timeout", strconv.Itoa(info.Build.GetCacheRequestTimeout()),
}
if url := cache.GetCacheDownloadURL(info.Build, cacheKey); url != nil {
args = append(args, "--url", url.String())
}
w.Noticef("Checking cache for %s...", cacheKey)
w.IfCmdWithOutput(info.RunnerCommand, args...)
w.Noticef("Successfully extracted cache")
w.Else()
w.Warningf("Failed to extract cache")
if cacheFallbackKey != "" {
b.addExtractCacheCommand(w, info, cacheFile, cacheFallbackKey, "")
}
w.EndIf()
}
func (b *AbstractShell) downloadArtifacts(w ShellWriter, job common.Dependency, info common.ShellScriptInfo) {
args := []string{
"artifacts-downloader",
"--url",
info.Build.Runner.URL,
"--token",
job.Token,
"--id",
strconv.Itoa(job.ID),
}
w.Noticef("Downloading artifacts for %s (%d)...", job.Name, job.ID)
w.Command(info.RunnerCommand, args...)
}
func (b *AbstractShell) jobArtifacts(info common.ShellScriptInfo) (otherJobs []common.Dependency) {
for _, otherJob := range info.Build.Dependencies {
if otherJob.ArtifactsFile.Filename == "" {
continue
}
otherJobs = append(otherJobs, otherJob)
}
return
}
func (b *AbstractShell) downloadAllArtifacts(w ShellWriter, info common.ShellScriptInfo) error {
otherJobs := b.jobArtifacts(info)
if len(otherJobs) == 0 {
return common.ErrSkipBuildStage
}
b.guardRunnerCommand(w, info.RunnerCommand, "Artifacts downloading", func() {
for _, otherJob := range otherJobs {
b.downloadArtifacts(w, otherJob, info)
}
})
return nil
}
func (b *AbstractShell) writePrepareScript(w ShellWriter, info common.ShellScriptInfo) error {
return nil
}
func (b *AbstractShell) writeGetSourcesScript(w ShellWriter, info common.ShellScriptInfo) error {
b.writeExports(w, info)
if !info.Build.IsSharedEnv() {
b.writeGitSSLConfig(w, info.Build, []string{"--global"})
}
if info.PreCloneScript != "" && info.Build.GetGitStrategy() != common.GitNone {
b.writeCommands(w, info.PreCloneScript)
}
if err := b.writeCloneFetchCmds(w, info); err != nil {
return err
}
return b.writeSubmoduleUpdateCmds(w, info)
}
func (b *AbstractShell) writeExports(w ShellWriter, info common.ShellScriptInfo) {
for _, variable := range info.Build.GetAllVariables() {
w.Variable(variable)
}
}
func (b *AbstractShell) writeGitSSLConfig(w ShellWriter, build *common.Build, where []string) {
repoURL, err := url.Parse(build.GetRemoteURL())
if err != nil {
w.Warningf("git SSL config: Can't parse repository URL. %s", err)
return
}
repoURL.Path = ""
repoURL.User = nil
host := repoURL.String()
variables := build.GetCITLSVariables()
args := append([]string{"config"}, where...)
for variable, config := range map[string]string{
tls.VariableCAFile: "sslCAInfo",
tls.VariableCertFile: "sslCert",
tls.VariableKeyFile: "sslKey",
} {
if variables.Get(variable) == "" {
continue
}
key := fmt.Sprintf("http.%s.%s", host, config)
w.Command("git", append(args, key, w.EnvVariableKey(variable))...)
}
}
func (b *AbstractShell) writeCloneFetchCmds(w ShellWriter, info common.ShellScriptInfo) error {
build := info.Build
// If LFS smudging was disabled by the user (by setting the GIT_LFS_SKIP_SMUDGE variable
// when defining the job) we're skipping this step.
//
// In other case we're disabling smudging here to prevent us from memory
// allocation failures.
//
// Please read https://gitlab.com/gitlab-org/gitlab-runner/issues/3366 and
// https://github.com/git-lfs/git-lfs/issues/3524 for context.
if !build.IsLFSSmudgeDisabled() {
w.Variable(common.JobVariable{Key: "GIT_LFS_SKIP_SMUDGE", Value: "1"})
}
err := b.handleGetSourcesStrategy(w, build)
if err != nil {
return err
}
if build.GetGitCheckout() {
b.writeCheckoutCmd(w, build)
// If LFS smudging was disabled by the user (by setting the GIT_LFS_SKIP_SMUDGE variable
// when defining the job) we're skipping this step.
//
// In other case, because we've disabled LFS smudging above, we need now manually call
// `git lfs pull` to fetch and checkout all LFS objects that may be present in
// the repository.
//
// Repositories without LFS objects (and without any LFS metadata) will be not
// affected by this command.
//
// Please read https://gitlab.com/gitlab-org/gitlab-runner/issues/3366 and
// https://github.com/git-lfs/git-lfs/issues/3524 for context.
if !build.IsLFSSmudgeDisabled() {
w.IfCmd("git", "lfs", "version")
w.Command("git", "lfs", "pull")
w.EmptyLine()
w.EndIf()
}
} else {
w.Noticef("Skipping Git checkout")
}
return nil
}
func (b *AbstractShell) handleGetSourcesStrategy(w ShellWriter, build *common.Build) error {
projectDir := build.FullProjectDir()
switch build.GetGitStrategy() {
case common.GitFetch:
b.writeRefspecFetchCmd(w, build, projectDir)
case common.GitClone:
w.RmDir(projectDir)
b.writeRefspecFetchCmd(w, build, projectDir)
case common.GitNone:
w.Noticef("Skipping Git repository setup")
w.MkDir(projectDir)
default:
return errors.New("unknown GIT_STRATEGY")
}
return nil
}
func (b *AbstractShell) writeRefspecFetchCmd(w ShellWriter, build *common.Build, projectDir string) {
depth := build.GitInfo.Depth
if depth > 0 {
w.Noticef("Fetching changes with git depth set to %d...", depth)
} else {
w.Noticef("Fetching changes...")
}
// initializing
templateDir := w.MkTmpDir("git-template")
templateFile := w.Join(templateDir, "config")
w.Command("git", "config", "-f", templateFile, "fetch.recurseSubmodules", "false")
if build.IsSharedEnv() {
b.writeGitSSLConfig(w, build, []string{"-f", templateFile})
}
b.writeGitCleanup(w, projectDir)
w.Command("git", "init", projectDir, "--template", templateDir)
w.Cd(projectDir)
// Add `git remote` or update existing
w.IfCmd("git", "remote", "add", "origin", build.GetRemoteURL())
w.Noticef("Created fresh repository.")
w.Else()
w.Command("git", "remote", "set-url", "origin", build.GetRemoteURL())
w.EndIf()
v := common.AppVersion
userAgent := fmt.Sprintf("http.userAgent=%s %s %s/%s", v.Name, v.Version, v.OS, v.Architecture)
fetchArgs := []string{"-c", userAgent, "fetch", "origin"}
fetchArgs = append(fetchArgs, build.GitInfo.Refspecs...)
if depth > 0 {
fetchArgs = append(fetchArgs, "--depth", strconv.Itoa(depth))
}
fetchArgs = append(fetchArgs, build.GetGitFetchFlags()...)
w.Command("git", fetchArgs...)
}
func (b *AbstractShell) writeGitCleanup(w ShellWriter, projectDir string) {
// Remove .git/{index,shallow,HEAD,config}.lock files from .git, which can fail the fetch command
// The file can be left if previous build was terminated during git operation
files := []string{
".git/index.lock",
".git/shallow.lock",
".git/HEAD.lock",
".git/hooks/post-checkout",
".git/config.lock",
}
for _, f := range files {
w.RmFile(path.Join(projectDir, f))
}
}
func (b *AbstractShell) writeCheckoutCmd(w ShellWriter, build *common.Build) {
w.Noticef("Checking out %s as %s...", build.GitInfo.Sha[0:8], build.GitInfo.Ref)
w.Command("git", "checkout", "-f", "-q", build.GitInfo.Sha)
cleanFlags := build.GetGitCleanFlags()
if len(cleanFlags) > 0 {
cleanArgs := append([]string{"clean"}, cleanFlags...)
w.Command("git", cleanArgs...)
}
}
func (b *AbstractShell) writeSubmoduleUpdateCmds(w ShellWriter, info common.ShellScriptInfo) error {
build := info.Build
switch build.GetSubmoduleStrategy() {
case common.SubmoduleNormal:
b.writeSubmoduleUpdateCmd(w, build, false)
case common.SubmoduleRecursive:
b.writeSubmoduleUpdateCmd(w, build, true)
case common.SubmoduleNone:
w.Noticef("Skipping Git submodules setup")
default:
return errors.New("unknown GIT_SUBMODULE_STRATEGY")
}
return nil
}
func (b *AbstractShell) writeSubmoduleUpdateCmd(w ShellWriter, build *common.Build, recursive bool) {
depth := build.GitInfo.Depth
b.writeSubmoduleUpdateNoticeMsg(w, recursive, depth)
var pathArgs []string
submodulePaths := strings.TrimSpace(build.GetSubmodulePaths())
if submodulePaths != "" {
pathArgs = append(pathArgs, "--", submodulePaths)
}
// Sync .git/config to .gitmodules in case URL changes (e.g. new build token)
args := []string{"submodule", "sync"}
if recursive {
args = append(args, "--recursive")
}
args = append(args, pathArgs...)
w.Command("git", args...)
// Update / initialize submodules
updateArgs := []string{"submodule", "update", "--init"}
foreachArgs := []string{"submodule", "foreach"}
if recursive {
updateArgs = append(updateArgs, "--recursive")
foreachArgs = append(foreachArgs, "--recursive")
}
if depth > 0 {
updateArgs = append(updateArgs, "--depth", strconv.Itoa(depth))
}
updateArgs = append(updateArgs, pathArgs...)
// Clean changed files in submodules
w.Command("git", append(foreachArgs, "git clean -ffxd")...)
w.Command("git", append(foreachArgs, "git reset --hard")...)
w.Command("git", updateArgs...)
// Clean changed files in sub-submodules
w.Command("git", append(foreachArgs, "git clean -ffxd")...)
if !build.IsLFSSmudgeDisabled() {
w.IfCmd("git", "lfs", "version")
w.Command("git", append(foreachArgs, "git lfs pull")...)
w.EndIf()
}
}
func (b *AbstractShell) writeSubmoduleUpdateNoticeMsg(w ShellWriter, recursive bool, depth int) {
switch {
case recursive && depth > 0:
w.Noticef("Updating/initializing submodules recursively with git depth set to %d...", depth)
case recursive && depth == 0:
w.Noticef("Updating/initializing submodules recursively...")
case depth > 0:
w.Noticef("Updating/initializing submodules with git depth set to %d...", depth)
default:
w.Noticef("Updating/initializing submodules...")
}
}
func (b *AbstractShell) writeRestoreCacheScript(w ShellWriter, info common.ShellScriptInfo) error {
b.writeExports(w, info)
b.writeCdBuildDir(w, info)
// Try to restore from main cache, if not found cache for default branch
return b.cacheExtractor(w, info)
}
func (b *AbstractShell) writeDownloadArtifactsScript(w ShellWriter, info common.ShellScriptInfo) error {
b.writeExports(w, info)
b.writeCdBuildDir(w, info)
return b.downloadAllArtifacts(w, info)
}
// Write the given string of commands using the provided ShellWriter object.
func (b *AbstractShell) writeCommands(w ShellWriter, commands ...string) {
for _, command := range commands {
command = strings.TrimSpace(command)
if command != "" {
lines := strings.SplitN(command, "\n", 2)
if len(lines) > 1 {
// TODO: this should be collapsable once we introduce that in GitLab
w.Noticef("$ %s # collapsed multi-line command", lines[0])
} else {
w.Noticef("$ %s", lines[0])
}
} else {
w.EmptyLine()
}
w.Line(command)
w.CheckForErrors()
}
}
func (b *AbstractShell) writeUserScript(
w ShellWriter,
info common.ShellScriptInfo,
buildStage common.BuildStage,
) error {
var scriptStep *common.Step
for _, step := range info.Build.Steps {
if common.StepToBuildStage(step) == buildStage {
scriptStep = &step
break
}
}
if scriptStep == nil {
return common.ErrSkipBuildStage
}
b.writeExports(w, info)
b.writeCdBuildDir(w, info)
if info.PreBuildScript != "" {
b.writeCommands(w, info.PreBuildScript)
}
b.writeCommands(w, scriptStep.Script...)
if info.PostBuildScript != "" {
b.writeCommands(w, info.PostBuildScript)
}
return nil
}
func (b *AbstractShell) cacheArchiver(w ShellWriter, info common.ShellScriptInfo, onSuccess bool) error {
b.writeExports(w, info)
b.writeCdBuildDir(w, info)
skipArchiveCache, err := b.archiveCache(w, info, onSuccess)
if err != nil {
return err
}
if skipArchiveCache {
return common.ErrSkipBuildStage
}
return nil
}
func (b *AbstractShell) archiveCache(w ShellWriter, info common.ShellScriptInfo, onSuccess bool) (bool, error) {
skipArchiveCache := true
for _, cacheOptions := range info.Build.Cache {
if !cacheOptions.When.ShouldCache(onSuccess) {
continue
}
// Create list of files to archive
archiverArgs := b.getArchiverArgs(cacheOptions)
if len(archiverArgs) < 1 {
// Skip creating archive
continue
}
skipArchiveCache = false
// Skip archiving if no cache is defined
cacheKey, cacheFile := b.cacheFile(info.Build, cacheOptions.Key)
if cacheKey == "" {
w.Noticef("Skipping cache archiving due to empty cache key")
continue
}
if ok, err := cacheOptions.CheckPolicy(common.CachePolicyPush); err != nil {
return false, fmt.Errorf("%w for %s", err, cacheKey)
} else if !ok {
w.Noticef("Not uploading cache %s due to policy", cacheKey)
continue
}
b.addCacheUploadCommand(w, info, cacheFile, archiverArgs, cacheKey)
}
return skipArchiveCache, nil
}
func (b *AbstractShell) getArchiverArgs(cacheOptions common.Cache) []string {
var archiverArgs []string
for _, path := range cacheOptions.Paths {
archiverArgs = append(archiverArgs, "--path", path)
}
if cacheOptions.Untracked {
archiverArgs = append(archiverArgs, "--untracked")
}
return archiverArgs
}
func (b *AbstractShell) addCacheUploadCommand(
w ShellWriter,
info common.ShellScriptInfo,
cacheFile string,
archiverArgs []string,
cacheKey string,
) {
args := []string{
"cache-archiver",
"--file", cacheFile,
"--timeout", strconv.Itoa(info.Build.GetCacheRequestTimeout()),
}
args = append(args, archiverArgs...)
// Generate cache upload address
args = append(args, getCacheUploadURL(info.Build, cacheKey)...)
env := cache.GetCacheUploadEnv(info.Build, cacheKey)
// Execute cache-archiver command. Failure is not fatal.
b.guardRunnerCommand(w, info.RunnerCommand, "Creating cache", func() {
w.Noticef("Creating cache %s...", cacheKey)
for key, value := range env {
w.Variable(common.JobVariable{Key: key, Value: value})
}
w.IfCmdWithOutput(info.RunnerCommand, args...)
w.Noticef("Created cache")
w.Else()
w.Warningf("Failed to create cache")
w.EndIf()
})
}
// getCacheUploadURL will first try to generate the GoCloud URL if it's
// available then fallback to a pre-signed URL.
func getCacheUploadURL(build *common.Build, cacheKey string) []string {
// Prefer Go Cloud URL if supported
goCloudURL := cache.GetCacheGoCloudURL(build, cacheKey)
if goCloudURL != nil {
return []string{"--gocloud-url", goCloudURL.String()}
}
uploadURL := cache.GetCacheUploadURL(build, cacheKey)
if uploadURL == nil {
return []string{}
}
urlArgs := []string{"--url", uploadURL.String()}
httpHeaders := cache.GetCacheUploadHeaders(build, cacheKey)
for key, values := range httpHeaders {
for _, value := range values {
urlArgs = append(urlArgs, "--header", fmt.Sprintf("%s: %s", key, value))
}
}
return urlArgs
}
func (b *AbstractShell) writeUploadArtifact(w ShellWriter, info common.ShellScriptInfo, artifact common.Artifact) bool {
args := []string{
"artifacts-uploader",
"--url",
info.Build.Runner.URL,
"--token",
info.Build.Token,
"--id",
strconv.Itoa(info.Build.ID),
}
// Create list of files to archive
var archiverArgs []string
for _, path := range artifact.Paths {
archiverArgs = append(archiverArgs, "--path", path)
}
// Create list of paths to be excluded from the archive
for _, path := range artifact.Exclude {
archiverArgs = append(archiverArgs, "--exclude", path)
}
if artifact.Untracked {
archiverArgs = append(archiverArgs, "--untracked")
}
if len(archiverArgs) < 1 {
// Skip creating archive
return false
}
args = append(args, archiverArgs...)
if artifact.Name != "" {
args = append(args, "--name", artifact.Name)
}
if artifact.ExpireIn != "" {
args = append(args, "--expire-in", artifact.ExpireIn)
}
if artifact.Format != "" {
args = append(args, "--artifact-format", string(artifact.Format))
}
if artifact.Type != "" {
args = append(args, "--artifact-type", artifact.Type)
}
b.guardRunnerCommand(w, info.RunnerCommand, "Uploading artifacts", func() {
w.Noticef("Uploading artifacts...")
w.Command(info.RunnerCommand, args...)
})
return true
}
func (b *AbstractShell) writeUploadArtifacts(w ShellWriter, info common.ShellScriptInfo, onSuccess bool) error {
if info.Build.Runner.URL == "" {
return common.ErrSkipBuildStage
}
b.writeExports(w, info)
b.writeCdBuildDir(w, info)
skipUploadArtifacts := true
for _, artifact := range info.Build.Artifacts {
if onSuccess && !artifact.When.OnSuccess() {
continue
}
if !onSuccess && !artifact.When.OnFailure() {
continue
}
if b.writeUploadArtifact(w, info, artifact) {
skipUploadArtifacts = false
}
}
if skipUploadArtifacts {
return common.ErrSkipBuildStage
}
return nil
}
func (b *AbstractShell) writeAfterScript(w ShellWriter, info common.ShellScriptInfo) error {
var afterScriptStep *common.Step
for _, step := range info.Build.Steps {
if step.Name == common.StepNameAfterScript {
afterScriptStep = &step
break
}
}
if afterScriptStep == nil || len(afterScriptStep.Script) == 0 {
return common.ErrSkipBuildStage
}
b.writeExports(w, info)
b.writeCdBuildDir(w, info)
w.Noticef("Running after script...")
b.writeCommands(w, afterScriptStep.Script...)
return nil
}
func (b *AbstractShell) writeUploadArtifactsOnSuccessScript(w ShellWriter, info common.ShellScriptInfo) error {
return b.writeUploadArtifacts(w, info, true)
}
func (b *AbstractShell) writeUploadArtifactsOnFailureScript(w ShellWriter, info common.ShellScriptInfo) error {
return b.writeUploadArtifacts(w, info, false)
}
func (b *AbstractShell) writeArchiveCacheOnSuccessScript(w ShellWriter, info common.ShellScriptInfo) error {
return b.cacheArchiver(w, info, true)
}
func (b *AbstractShell) writeArchiveCacheOnFailureScript(w ShellWriter, info common.ShellScriptInfo) error {
return b.cacheArchiver(w, info, false)
}
func (b *AbstractShell) writeCleanupFileVariablesScript(w ShellWriter, info common.ShellScriptInfo) error {
skipCleanupFileVariables := true
for _, variable := range info.Build.GetAllVariables() {
if !variable.File {
continue
}
skipCleanupFileVariables = false
w.RmFile(w.TmpFile(variable.Key))
}
if skipCleanupFileVariables {
return common.ErrSkipBuildStage
}
return nil
}
func (b *AbstractShell) writeScript(w ShellWriter, buildStage common.BuildStage, info common.ShellScriptInfo) error {
methods := map[common.BuildStage]func(ShellWriter, common.ShellScriptInfo) error{
common.BuildStagePrepare: b.writePrepareScript,
common.BuildStageGetSources: b.writeGetSourcesScript,
common.BuildStageRestoreCache: b.writeRestoreCacheScript,
common.BuildStageDownloadArtifacts: b.writeDownloadArtifactsScript,
common.BuildStageAfterScript: b.writeAfterScript,
common.BuildStageArchiveOnSuccessCache: b.writeArchiveCacheOnSuccessScript,
common.BuildStageArchiveOnFailureCache: b.writeArchiveCacheOnFailureScript,
common.BuildStageUploadOnSuccessArtifacts: b.writeUploadArtifactsOnSuccessScript,
common.BuildStageUploadOnFailureArtifacts: b.writeUploadArtifactsOnFailureScript,
common.BuildStageCleanupFileVariables: b.writeCleanupFileVariablesScript,
}
fn, ok := methods[buildStage]
if !ok {
return b.writeUserScript(w, info, buildStage)
}
return fn(w, info)
}
package shells
import (
"bufio"
"bytes"
"fmt"
"io"
"path"
"runtime"
"strconv"
"strings"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
)
const BashDetectShellScript = `if [ -x /usr/local/bin/bash ]; then
exec /usr/local/bin/bash $@
elif [ -x /usr/bin/bash ]; then
exec /usr/bin/bash $@
elif [ -x /bin/bash ]; then
exec /bin/bash $@
elif [ -x /usr/local/bin/sh ]; then
exec /usr/local/bin/sh $@
elif [ -x /usr/bin/sh ]; then
exec /usr/bin/sh $@
elif [ -x /bin/sh ]; then
exec /bin/sh $@
elif [ -x /busybox/sh ]; then
exec /busybox/sh $@
else
echo shell not found
exit 1
fi
`
type BashShell struct {
AbstractShell
Shell string
}
type BashWriter struct {
bytes.Buffer
TemporaryPath string
Shell string
indent int
checkForErrors bool
useNewEval bool
}
func (b *BashWriter) GetTemporaryPath() string {
return b.TemporaryPath
}
func (b *BashWriter) Line(text string) {
b.WriteString(strings.Repeat(" ", b.indent) + text + "\n")
}
func (b *BashWriter) Linef(format string, arguments ...interface{}) {
b.Line(fmt.Sprintf(format, arguments...))
}
func (b *BashWriter) CheckForErrors() {
if !b.checkForErrors {
return
}
b.Line("_runner_exit_code=$?; if [[ $_runner_exit_code -ne 0 ]]; then exit $_runner_exit_code; fi")
}
func (b *BashWriter) Indent() {
b.indent++
}
func (b *BashWriter) Unindent() {
b.indent--
}
func (b *BashWriter) Command(command string, arguments ...string) {
b.Line(b.buildCommand(command, arguments...))
b.CheckForErrors()
}
func (b *BashWriter) buildCommand(command string, arguments ...string) string {
list := []string{
helpers.ShellEscape(command),
}
for _, argument := range arguments {
list = append(list, strconv.Quote(argument))
}
return strings.Join(list, " ")
}
func (b *BashWriter) TmpFile(name string) string {
return b.Absolute(path.Join(b.TemporaryPath, name))
}
func (b *BashWriter) EnvVariableKey(name string) string {
return fmt.Sprintf("$%s", name)
}
func (b *BashWriter) Variable(variable common.JobVariable) {
if variable.File {
variableFile := b.TmpFile(variable.Key)
b.Linef("mkdir -p %q", helpers.ToSlash(b.TemporaryPath))
b.Linef("echo -n %s > %q", helpers.ShellEscape(variable.Value), variableFile)
b.Linef("export %s=%q", helpers.ShellEscape(variable.Key), variableFile)
} else {
b.Linef("export %s=%s", helpers.ShellEscape(variable.Key), helpers.ShellEscape(variable.Value))
}
}
func (b *BashWriter) IfDirectory(path string) {
b.Linef("if [[ -d %q ]]; then", path)
b.Indent()
}
func (b *BashWriter) IfFile(path string) {
b.Linef("if [[ -e %q ]]; then", path)
b.Indent()
}
func (b *BashWriter) IfCmd(cmd string, arguments ...string) {
cmdline := b.buildCommand(cmd, arguments...)
b.Linef("if %s >/dev/null 2>/dev/null; then", cmdline)
b.Indent()
}
func (b *BashWriter) IfCmdWithOutput(cmd string, arguments ...string) {
cmdline := b.buildCommand(cmd, arguments...)
b.Linef("if %s; then", cmdline)
b.Indent()
}
func (b *BashWriter) Else() {
b.Unindent()
b.Line("else")
b.Indent()
}
func (b *BashWriter) EndIf() {
b.Unindent()
b.Line("fi")
}
func (b *BashWriter) Cd(path string) {
b.Command("cd", path)
}
func (b *BashWriter) MkDir(path string) {
b.Command("mkdir", "-p", path)
}
func (b *BashWriter) MkTmpDir(name string) string {
path := path.Join(b.TemporaryPath, name)
b.MkDir(path)
return path
}
func (b *BashWriter) RmDir(path string) {
b.Command("rm", "-r", "-f", path)
}
func (b *BashWriter) RmFile(path string) {
b.Command("rm", "-f", path)
}
func (b *BashWriter) Absolute(dir string) string {
if path.IsAbs(dir) {
return dir
}
return path.Join("$PWD", dir)
}
func (b *BashWriter) Join(elem ...string) string {
return path.Join(elem...)
}
func (b *BashWriter) Printf(format string, arguments ...interface{}) {
coloredText := helpers.ANSI_RESET + fmt.Sprintf(format, arguments...)
b.Line("echo " + helpers.ShellEscape(coloredText))
}
func (b *BashWriter) Noticef(format string, arguments ...interface{}) {
coloredText := helpers.ANSI_BOLD_GREEN + fmt.Sprintf(format, arguments...) + helpers.ANSI_RESET
b.Line("echo " + helpers.ShellEscape(coloredText))
}
func (b *BashWriter) Warningf(format string, arguments ...interface{}) {
coloredText := helpers.ANSI_YELLOW + fmt.Sprintf(format, arguments...) + helpers.ANSI_RESET
b.Line("echo " + helpers.ShellEscape(coloredText))
}
func (b *BashWriter) Errorf(format string, arguments ...interface{}) {
coloredText := helpers.ANSI_BOLD_RED + fmt.Sprintf(format, arguments...) + helpers.ANSI_RESET
b.Line("echo " + helpers.ShellEscape(coloredText))
}
func (b *BashWriter) EmptyLine() {
b.Line("echo")
}
func (b *BashWriter) Finish(trace bool) string {
var buffer bytes.Buffer
w := bufio.NewWriter(&buffer)
b.writeShebang(w)
b.writeTrace(w, trace)
b.writeScript(w)
_ = w.Flush()
return buffer.String()
}
func (b *BashWriter) writeShebang(w io.Writer) {
if b.Shell != "" {
_, _ = io.WriteString(w, "#!/usr/bin/env "+b.Shell+"\n\n")
}
}
func (b *BashWriter) writeTrace(w io.Writer, trace bool) {
if trace {
_, _ = io.WriteString(w, "set -o xtrace\n")
}
}
func (b *BashWriter) writeEval(w io.Writer) {
command := ": | eval " + helpers.ShellEscape(b.String()) + "\n"
if b.useNewEval {
command = ": | (eval " + helpers.ShellEscape(b.String()) + ")\n"
}
_, _ = io.WriteString(w, command)
}
func (b *BashWriter) writeScript(w io.Writer) {
_, _ = io.WriteString(w, "set -eo pipefail\n")
_, _ = io.WriteString(w, "set +o noclobber\n")
b.writeEval(w)
_, _ = io.WriteString(w, "exit 0\n")
}
func (b *BashShell) GetName() string {
return b.Shell
}
func (b *BashShell) GetConfiguration(info common.ShellScriptInfo) (*common.ShellConfiguration, error) {
var detectScript string
var shellCommand string
if info.Type == common.LoginShell {
detectScript = strings.ReplaceAll(BashDetectShellScript, "$@", "--login")
shellCommand = b.Shell + " --login"
} else {
detectScript = strings.ReplaceAll(BashDetectShellScript, "$@", "")
shellCommand = b.Shell
}
script := &common.ShellConfiguration{}
script.DockerCommand = []string{"sh", "-c", detectScript}
// su
if info.User != "" {
script.Command = "su"
if runtime.GOOS == "linux" {
script.Arguments = append(script.Arguments, "-s", "/bin/"+b.Shell)
}
script.Arguments = append(
script.Arguments,
info.User,
"-c", shellCommand,
)
} else {
script.Command = b.Shell
if info.Type == common.LoginShell {
script.Arguments = append(script.Arguments, "--login")
}
}
return script, nil
}
func (b *BashShell) GenerateScript(buildStage common.BuildStage, info common.ShellScriptInfo) (string, error) {
w := &BashWriter{
TemporaryPath: info.Build.TmpProjectDir(),
Shell: b.Shell,
checkForErrors: info.Build.IsFeatureFlagOn(featureflags.EnableBashExitCodeCheck),
useNewEval: info.Build.IsFeatureFlagOn(featureflags.UseNewEvalStrategy),
}
return b.generateScript(w, buildStage, info)
}
func (b *BashShell) generateScript(
w ShellWriter,
buildStage common.BuildStage,
info common.ShellScriptInfo,
) (string, error) {
b.ensurePrepareStageHostnameMessage(w, buildStage, info)
err := b.writeScript(w, buildStage, info)
script := w.Finish(info.Build.IsDebugTraceEnabled())
return script, err
}
func (b *BashShell) ensurePrepareStageHostnameMessage(
w ShellWriter,
buildStage common.BuildStage,
info common.ShellScriptInfo,
) {
if buildStage == common.BuildStagePrepare {
if info.Build.Hostname != "" {
w.Line("echo " + strconv.Quote("Running on $(hostname) via "+info.Build.Hostname+"..."))
} else {
w.Line("echo " + strconv.Quote("Running on $(hostname)..."))
}
}
}
func (b *BashShell) IsDefault() bool {
return runtime.GOOS != OSWindows && b.Shell == "bash"
}
func init() {
common.RegisterShell(&BashShell{Shell: "sh"})
common.RegisterShell(&BashShell{Shell: "bash"})
}
package shells
import (
"bufio"
"bytes"
"fmt"
"io"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
// BashTrapShellScript is used to wrap a shell script in a trap that makes sure the script always exits
// with exit code of 0 this can be useful in container environments where exiting with an exit code different from 0
// would kill the container.
// At the same time it writes to a file the actual exit code of the script as well as the filename
// of the script as json.
const bashTrapShellScript = `runner_script_trap() {
exit_code=$?
log_file=%s
out_json="{\"command_exit_code\": $exit_code, \"script\": \"$0\"}"
# Make sure the command status will always be printed on a new line
if [[ $(tail -c1 $log_file | wc -l) -gt 0 ]]; then
printf "$out_json\n" >> $log_file
else
printf "\n$out_json\n" >> $log_file
fi
exit 0
}
trap runner_script_trap EXIT
`
type BashTrapShellWriter struct {
*BashWriter
logFile string
}
func (b *BashTrapShellWriter) Finish(trace bool) string {
var buffer bytes.Buffer
w := bufio.NewWriter(&buffer)
b.writeShebang(w)
b.writeTrap(w)
b.writeTrace(w, trace)
b.writeScript(w)
_ = w.Flush()
return buffer.String()
}
func (b *BashTrapShellWriter) writeTrap(w io.Writer) {
_, _ = fmt.Fprintf(w, bashTrapShellScript, b.logFile)
}
type BashTrapShell struct {
*BashShell
LogFile string
}
func (b *BashTrapShell) GenerateScript(buildStage common.BuildStage, info common.ShellScriptInfo) (string, error) {
w := &BashTrapShellWriter{
BashWriter: &BashWriter{
TemporaryPath: info.Build.TmpProjectDir(),
Shell: b.Shell,
},
logFile: b.LogFile,
}
return b.generateScript(w, buildStage, info)
}
package shells
import (
"bufio"
"bytes"
"fmt"
"io"
"path"
"path/filepath"
"runtime"
"strings"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
)
type CmdShell struct {
AbstractShell
}
type CmdWriter struct {
bytes.Buffer
TemporaryPath string
indent int
disableDelayedErrorLevelExpansion bool
}
func batchQuote(text string) string {
return "\"" + batchEscapeInsideQuotedString(text) + "\""
}
func batchEscapeInsideQuotedString(text string) string {
// taken from: http://www.robvanderwoude.com/escapechars.php
text = strings.ReplaceAll(text, "^", "^^")
text = strings.ReplaceAll(text, "!", "^^!")
text = strings.ReplaceAll(text, "&", "^&")
text = strings.ReplaceAll(text, "<", "^<")
text = strings.ReplaceAll(text, ">", "^>")
text = strings.ReplaceAll(text, "|", "^|")
text = strings.ReplaceAll(text, "\r", "")
text = strings.ReplaceAll(text, "\n", "!nl!")
return text
}
func batchEscapeVariable(text string) string {
text = strings.ReplaceAll(text, "%", "%%")
text = batchEscape(text)
return text
}
// If not inside a quoted string (e.g., echo text), escape more things
func batchEscape(text string) string {
text = batchEscapeInsideQuotedString(text)
text = strings.ReplaceAll(text, "(", "^(")
text = strings.ReplaceAll(text, ")", "^)")
return text
}
func (b *CmdShell) GetName() string {
return "cmd"
}
func (b *CmdWriter) GetTemporaryPath() string {
return b.TemporaryPath
}
func (b *CmdWriter) Line(text string) {
b.WriteString(strings.Repeat(" ", b.indent) + text + "\r\n")
}
func (b *CmdWriter) Linef(format string, arguments ...interface{}) {
b.Line(fmt.Sprintf(format, arguments...))
}
func (b *CmdWriter) CheckForErrors() {
b.checkErrorLevel()
}
func (b *CmdWriter) Indent() {
b.indent++
}
func (b *CmdWriter) Unindent() {
b.indent--
}
func (b *CmdWriter) checkErrorLevel() {
errCheck := "IF !errorlevel! NEQ 0 exit /b !errorlevel!"
b.Line(b.updateErrLevelCheck(errCheck))
b.Line("")
}
func (b *CmdWriter) updateErrLevelCheck(errCheck string) string {
if b.disableDelayedErrorLevelExpansion {
return strings.ReplaceAll(errCheck, "!", "%")
}
return errCheck
}
func (b *CmdWriter) Command(command string, arguments ...string) {
b.Line(b.buildCommand(command, arguments...))
b.checkErrorLevel()
}
func (b *CmdWriter) buildCommand(command string, arguments ...string) string {
list := []string{
batchQuote(command),
}
for _, argument := range arguments {
list = append(list, batchQuote(argument))
}
return strings.Join(list, " ")
}
func (b *CmdWriter) TmpFile(name string) string {
filePath := b.Absolute(path.Join(b.TemporaryPath, name))
return helpers.ToBackslash(filePath)
}
func (b *CmdWriter) EnvVariableKey(name string) string {
return fmt.Sprintf("%%%s%%", name)
}
func (b *CmdWriter) Variable(variable common.JobVariable) {
if variable.File {
variableFile := b.TmpFile(variable.Key)
b.Linef("md %q 2>NUL 1>NUL", batchEscape(helpers.ToBackslash(b.TemporaryPath)))
b.Linef("echo %s > %s", batchEscapeVariable(variable.Value), batchEscape(variableFile))
b.Linef("SET %s=%s", batchEscapeVariable(variable.Key), batchEscape(variableFile))
} else {
b.Linef("SET %s=%s", batchEscapeVariable(variable.Key), batchEscapeVariable(variable.Value))
}
}
func (b *CmdWriter) IfDirectory(path string) {
b.Linef("IF EXIST %s (", batchQuote(helpers.ToBackslash(path)))
b.Indent()
}
func (b *CmdWriter) IfFile(path string) {
b.Linef("IF EXIST %s (", batchQuote(helpers.ToBackslash(path)))
b.Indent()
}
func (b *CmdWriter) IfCmd(cmd string, arguments ...string) {
cmdline := b.buildCommand(cmd, arguments...)
b.Linef("%s 2>NUL 1>NUL", cmdline)
errCheck := "IF !errorlevel! EQU 0 ("
b.Line(b.updateErrLevelCheck(errCheck))
b.Indent()
}
func (b *CmdWriter) IfCmdWithOutput(cmd string, arguments ...string) {
cmdline := b.buildCommand(cmd, arguments...)
b.Line(cmdline)
errCheck := "IF !errorlevel! EQU 0 ("
b.Line(b.updateErrLevelCheck(errCheck))
b.Indent()
}
func (b *CmdWriter) Else() {
b.Unindent()
b.Line(") ELSE (")
b.Indent()
}
func (b *CmdWriter) EndIf() {
b.Unindent()
b.Line(")")
}
func (b *CmdWriter) Cd(path string) {
b.Line("cd /D " + batchQuote(helpers.ToBackslash(path)))
b.checkErrorLevel()
}
func (b *CmdWriter) MkDir(path string) {
args := batchQuote(helpers.ToBackslash(path)) + " 2>NUL 1>NUL"
b.Linef("dir %s || md %s", args, args)
}
func (b *CmdWriter) MkTmpDir(name string) string {
path := helpers.ToBackslash(path.Join(b.TemporaryPath, name))
b.MkDir(path)
return path
}
func (b *CmdWriter) RmDir(path string) {
b.Linef("rd /s /q %s 2>NUL 1>NUL", batchQuote(helpers.ToBackslash(path)))
}
func (b *CmdWriter) RmFile(path string) {
b.Linef("del /f /q %s 2>NUL 1>NUL", batchQuote(helpers.ToBackslash(path)))
}
func (b *CmdWriter) Printf(format string, arguments ...interface{}) {
coloredText := helpers.ANSI_RESET + fmt.Sprintf(format, arguments...) + helpers.ANSI_RESET
b.Line("echo " + batchEscapeVariable(coloredText))
}
func (b *CmdWriter) Noticef(format string, arguments ...interface{}) {
coloredText := helpers.ANSI_BOLD_GREEN + fmt.Sprintf(format, arguments...) + helpers.ANSI_RESET
b.Line("echo " + batchEscapeVariable(coloredText))
}
func (b *CmdWriter) Warningf(format string, arguments ...interface{}) {
coloredText := helpers.ANSI_YELLOW + fmt.Sprintf(format, arguments...) + helpers.ANSI_RESET
b.Line("echo " + batchEscapeVariable(coloredText))
}
func (b *CmdWriter) Errorf(format string, arguments ...interface{}) {
coloredText := helpers.ANSI_BOLD_RED + fmt.Sprintf(format, arguments...) + helpers.ANSI_RESET
b.Line("echo " + batchEscapeVariable(coloredText))
}
func (b *CmdWriter) EmptyLine() {
b.Line("echo.")
}
func (b *CmdWriter) Absolute(dir string) string {
if filepath.IsAbs(dir) {
return dir
}
return filepath.Join("%CD%", dir)
}
func (b *CmdWriter) Join(elem ...string) string {
newPath := path.Join(elem...)
return helpers.ToBackslash(newPath)
}
func (b *CmdWriter) Finish(trace bool) string {
var buffer bytes.Buffer
w := bufio.NewWriter(&buffer)
if trace {
_, _ = io.WriteString(w, "@echo on\r\n")
} else {
_, _ = io.WriteString(w, "@echo off\r\n")
}
_, _ = io.WriteString(w, "setlocal enableextensions\r\n")
_, _ = io.WriteString(w, "setlocal enableDelayedExpansion\r\n")
_, _ = io.WriteString(w, "set nl=^\r\n\r\n\r\n")
_, _ = io.WriteString(w, b.String())
_ = w.Flush()
return buffer.String()
}
func (b *CmdShell) GetConfiguration(info common.ShellScriptInfo) (script *common.ShellConfiguration, err error) {
script = &common.ShellConfiguration{
Command: "cmd",
Arguments: []string{"/C"},
PassFile: true,
Extension: "cmd",
}
return
}
func (b *CmdShell) GenerateScript(
buildStage common.BuildStage,
info common.ShellScriptInfo,
) (script string, err error) {
//nolint:lll
w := &CmdWriter{
TemporaryPath: info.Build.TmpProjectDir(),
disableDelayedErrorLevelExpansion: info.Build.IsFeatureFlagOn(featureflags.CmdDisableDelayedErrorLevelExpansion),
}
if buildStage == common.BuildStagePrepare {
if info.Build.Hostname != "" {
w.Line("echo Running on %COMPUTERNAME% via " + batchEscape(info.Build.Hostname) + "...")
} else {
w.Line("echo Running on %COMPUTERNAME%...")
}
w.Warningf("DEPRECATION: CMD shell is deprecated and will no longer be supported")
}
err = b.writeScript(w, buildStage, info)
script = w.Finish(info.Build.IsDebugTraceEnabled())
return
}
func (b *CmdShell) IsDefault() bool {
return runtime.GOOS == OSWindows
}
func init() {
common.RegisterShell(&CmdShell{})
}
package shells
import (
"bufio"
"bytes"
"fmt"
"io"
"os"
"path"
"path/filepath"
"runtime"
"strings"
"gitlab.com/gitlab-org/gitlab-runner/common"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
)
const (
kubernetesExecutor = "kubernetes"
dockerExecutor = "docker"
dockerWindowsExecutor = "docker-windows"
SNPwsh = "pwsh"
SNPowershell = "powershell"
// Before executing a script, powershell parses it.
// A `ParserError` can then be thrown if a parsing error is found.
// Those errors are not catched by the powershell_trap_script thus causing the job to hang
// To avoid this problem, the PwshValidationScript is used to validate the given script and eventually to cause
// the job to fail if a `ParserError` is thrown
PwshValidationScript = `
param (
[Parameter(Mandatory=$true,Position=1)]
[string]$Path,
[Parameter(Mandatory=$true,Position=2)]
[string]$LogFile
)
# Empty collection for errors
$Errors = @()
$input = [IO.File]::ReadAllText($Path)
[void][System.Management.Automation.Language.Parser]::ParseInput($input,[ref]$null,[ref]$Errors)
if($Errors.Count -gt 0){
foreach ($err in $Errors) { Write-Error $err.toString() }
$out_json= '{"command_exit_code":1, "script": "' + $MyInvocation.MyCommand.Name + '"}'
Add-Content $LogFile @"
$out_json
"@
exit 0
}
pwsh -File $Path
`
)
type PowerShell struct {
AbstractShell
Shell string
EOL string
}
type PsWriter struct {
bytes.Buffer
TemporaryPath string
indent int
Shell string
EOL string
resolvePaths bool
}
func stdinCmdArgs() []string {
return []string{
"-NoProfile",
"-NoLogo",
"-InputFormat",
"text",
"-OutputFormat",
"text",
"-NonInteractive",
"-ExecutionPolicy",
"Bypass",
"-Command",
"-",
}
}
func fileCmdArgs() []string {
return []string{"-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command"}
}
func PowershellDockerCmd(shell string) []string {
return append([]string{shell}, stdinCmdArgs()...)
}
func psQuote(text string) string {
// taken from: http://www.robvanderwoude.com/escapechars.php
text = strings.ReplaceAll(text, "`", "``")
// text = strings.ReplaceAll(text, "\0", "`0")
text = strings.ReplaceAll(text, "\a", "`a")
text = strings.ReplaceAll(text, "\b", "`b")
text = strings.ReplaceAll(text, "\f", "^f")
text = strings.ReplaceAll(text, "\r", "`r")
text = strings.ReplaceAll(text, "\n", "`n")
text = strings.ReplaceAll(text, "\t", "^t")
text = strings.ReplaceAll(text, "\v", "^v")
text = strings.ReplaceAll(text, "#", "`#")
text = strings.ReplaceAll(text, "'", "`'")
text = strings.ReplaceAll(text, "\"", "`\"")
return `"` + text + `"`
}
func psQuoteVariable(text string) string {
text = psQuote(text)
text = strings.ReplaceAll(text, "$", "`$")
text = strings.ReplaceAll(text, "``e", "`e")
return text
}
func (p *PsWriter) GetTemporaryPath() string {
return p.TemporaryPath
}
func (p *PsWriter) Line(text string) {
p.WriteString(strings.Repeat(" ", p.indent) + text + p.EOL)
}
func (p *PsWriter) Linef(format string, arguments ...interface{}) {
p.Line(fmt.Sprintf(format, arguments...))
}
func (p *PsWriter) CheckForErrors() {
p.checkErrorLevel()
}
func (p *PsWriter) Indent() {
p.indent++
}
func (p *PsWriter) Unindent() {
p.indent--
}
func (p *PsWriter) checkErrorLevel() {
p.Line("if(!$?) { Exit &{if($LASTEXITCODE) {$LASTEXITCODE} else {1}} }")
p.Line("")
}
func (p *PsWriter) Command(command string, arguments ...string) {
p.Line(p.buildCommand(command, arguments...))
p.checkErrorLevel()
}
func (p *PsWriter) buildCommand(command string, arguments ...string) string {
list := []string{
psQuote(command),
}
for _, argument := range arguments {
list = append(list, psQuote(argument))
}
return "& " + strings.Join(list, " ")
}
func (p *PsWriter) resolvePath(path string) string {
if p.resolvePaths {
return fmt.Sprintf("$ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath(%s)", psQuote(path))
}
return psQuote(p.fromSlash(path))
}
func (p *PsWriter) TmpFile(name string) string {
if p.resolvePaths {
return p.Join(p.TemporaryPath, name)
}
filePath := p.Absolute(p.Join(p.TemporaryPath, name))
return p.fromSlash(filePath)
}
func (p *PsWriter) fromSlash(path string) string {
if p.resolvePaths {
return path
}
if p.Shell == SNPwsh {
// pwsh wants OS slash style, not necessarily backslashes
return filepath.FromSlash(path)
}
return helpers.ToBackslash(path)
}
func (p *PsWriter) EnvVariableKey(name string) string {
return fmt.Sprintf("$%s", name)
}
func (p *PsWriter) Variable(variable common.JobVariable) {
if variable.File {
variableFile := p.TmpFile(variable.Key)
p.MkDir(p.TemporaryPath)
p.Linef(
"[System.IO.File]::WriteAllText(%s, %s)",
p.resolvePath(variableFile),
psQuoteVariable(variable.Value),
)
p.Linef("$%s=%s", variable.Key, p.resolvePath(variableFile))
} else {
p.Linef("$%s=%s", variable.Key, psQuoteVariable(variable.Value))
}
p.Linef("$env:%s=$%s", variable.Key, variable.Key)
}
func (p *PsWriter) IfDirectory(path string) {
p.Linef("if(Test-Path %s -PathType Container) {", p.resolvePath(path))
p.Indent()
}
func (p *PsWriter) IfFile(path string) {
p.Linef("if(Test-Path %s -PathType Leaf) {", p.resolvePath(path))
p.Indent()
}
func (p *PsWriter) IfCmd(cmd string, arguments ...string) {
p.ifInTryCatch(p.buildCommand(cmd, arguments...) + " 2>$null")
}
func (p *PsWriter) IfCmdWithOutput(cmd string, arguments ...string) {
p.ifInTryCatch(p.buildCommand(cmd, arguments...))
}
func (p *PsWriter) ifInTryCatch(cmd string) {
p.Line("Set-Variable -Name cmdErr -Value $false")
p.Line("Try {")
p.Indent()
p.Line(cmd)
p.Line("if(!$?) { throw &{if($LASTEXITCODE) {$LASTEXITCODE} else {1}} }")
p.Unindent()
p.Line("} Catch {")
p.Indent()
p.Line("Set-Variable -Name cmdErr -Value $true")
p.Unindent()
p.Line("}")
p.Line("if(!$cmdErr) {")
p.Indent()
}
func (p *PsWriter) Else() {
p.Unindent()
p.Line("} else {")
p.Indent()
}
func (p *PsWriter) EndIf() {
p.Unindent()
p.Line("}")
}
func (p *PsWriter) Cd(path string) {
p.Line("cd " + p.resolvePath(path))
p.checkErrorLevel()
}
func (p *PsWriter) MkDir(path string) {
p.Linef("New-Item -ItemType directory -Force -Path %s | out-null", p.resolvePath(path))
}
func (p *PsWriter) MkTmpDir(name string) string {
dirPath := p.Join(p.TemporaryPath, name)
p.MkDir(dirPath)
return dirPath
}
func (p *PsWriter) RmDir(path string) {
path = p.resolvePath(path)
p.Linef(
"if( (Get-Command -Name Remove-Item2 -Module NTFSSecurity -ErrorAction SilentlyContinue) "+
"-and (Test-Path %s -PathType Container) ) {",
path,
)
p.Indent()
p.Line("Remove-Item2 -Force -Recurse " + path)
p.Unindent()
p.Linef("} elseif(Test-Path %s) {", path)
p.Indent()
p.Line("Remove-Item -Force -Recurse " + path)
p.Unindent()
p.Line("}")
p.Line("")
}
func (p *PsWriter) RmFile(path string) {
path = p.resolvePath(path)
p.Line(
"if( (Get-Command -Name Remove-Item2 -Module NTFSSecurity -ErrorAction SilentlyContinue) " +
"-and (Test-Path " + path + " -PathType Leaf) ) {")
p.Indent()
p.Line("Remove-Item2 -Force " + path)
p.Unindent()
p.Linef("} elseif(Test-Path %s) {", path)
p.Indent()
p.Line("Remove-Item -Force " + path)
p.Unindent()
p.Line("}")
p.Line("")
}
func (p *PsWriter) Printf(format string, arguments ...interface{}) {
coloredText := helpers.ANSI_RESET + fmt.Sprintf(format, arguments...)
p.Line("echo " + psQuoteVariable(coloredText))
}
func (p *PsWriter) Noticef(format string, arguments ...interface{}) {
coloredText := helpers.ANSI_BOLD_GREEN + fmt.Sprintf(format, arguments...) + helpers.ANSI_RESET
p.Line("echo " + psQuoteVariable(coloredText))
}
func (p *PsWriter) Warningf(format string, arguments ...interface{}) {
coloredText := helpers.ANSI_YELLOW + fmt.Sprintf(format, arguments...) + helpers.ANSI_RESET
p.Line("echo " + psQuoteVariable(coloredText))
}
func (p *PsWriter) Errorf(format string, arguments ...interface{}) {
coloredText := helpers.ANSI_BOLD_RED + fmt.Sprintf(format, arguments...) + helpers.ANSI_RESET
p.Line("echo " + psQuoteVariable(coloredText))
}
func (p *PsWriter) EmptyLine() {
p.Line(`echo ""`)
}
func (p *PsWriter) Absolute(dir string) string {
if p.resolvePaths {
return dir
}
if filepath.IsAbs(dir) {
return dir
}
p.Linef("$CurrentDirectory = (Resolve-Path .%s).Path", string(os.PathSeparator))
return p.Join("$CurrentDirectory", dir)
}
func (p *PsWriter) Join(elem ...string) string {
if p.resolvePaths {
// We rely on the resolve function and always use forward slashes
// when joining paths.
return path.Join(elem...)
}
return filepath.Join(elem...)
}
func (p *PsWriter) Finish(trace bool) string {
var buffer bytes.Buffer
w := bufio.NewWriter(&buffer)
if p.Shell != SNPwsh {
// write UTF-8 BOM (Powershell Core doesn't use a BOM as mentioned in
// https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3896#note_157830131)
_, _ = io.WriteString(w, "\xef\xbb\xbf")
}
p.writeTrace(w, trace)
if p.Shell == SNPwsh {
_, _ = io.WriteString(w, `$ErrorActionPreference = "Stop"`+p.EOL+p.EOL)
}
// add empty line to close code-block when it is piped to STDIN
p.Line("")
_, _ = io.WriteString(w, p.String())
_ = w.Flush()
return buffer.String()
}
func (p *PsWriter) writeShebang(w io.Writer) {
if p.Shell != "" {
_, _ = io.WriteString(w, "#!/usr/bin/env "+p.Shell+p.EOL+p.EOL)
}
}
func (p *PsWriter) writeTrace(w io.Writer, trace bool) {
if trace {
_, _ = io.WriteString(w, "Set-PSDebug -Trace 2"+p.EOL)
}
}
func (p *PsWriter) writeScript(w io.Writer) {
lines := strings.Split(p.String(), p.EOL)
_, _ = io.WriteString(w, strings.Join(lines, p.EOL))
_, _ = io.WriteString(w, p.EOL+p.EOL+"trap {runner_script_trap} runner_script_trap"+p.EOL+p.EOL+"exit 0"+p.EOL)
}
func (b *PowerShell) GetName() string {
return b.Shell
}
func (b *PowerShell) GetConfiguration(info common.ShellScriptInfo) (*common.ShellConfiguration, error) {
script := &common.ShellConfiguration{
Command: b.Shell,
Arguments: stdinCmdArgs(),
PassFile: !b.isStdinSupported(info),
Extension: "ps1",
DockerCommand: PowershellDockerCmd(b.Shell),
}
if script.PassFile {
script.Arguments = fileCmdArgs()
}
return script, nil
}
func (b *PowerShell) isStdinSupported(info common.ShellScriptInfo) bool {
executor := info.Build.Runner.Executor
return executor == kubernetesExecutor ||
executor == dockerExecutor ||
executor == dockerWindowsExecutor
}
func (b *PowerShell) GenerateScript(buildStage common.BuildStage, info common.ShellScriptInfo) (string, error) {
w := &PsWriter{
Shell: b.Shell,
EOL: b.EOL,
TemporaryPath: info.Build.TmpProjectDir(),
resolvePaths: info.Build.IsFeatureFlagOn(featureflags.UsePowershellPathResolver),
}
return b.generateScript(w, buildStage, info)
}
func (b *PowerShell) generateScript(
w ShellWriter,
buildStage common.BuildStage,
info common.ShellScriptInfo,
) (string, error) {
b.ensurePrepareStageHostnameMessage(w, buildStage, info)
err := b.writeScript(w, buildStage, info)
if err != nil {
return "", err
}
script := w.Finish(info.Build.IsDebugTraceEnabled())
return script, nil
}
func (b *PowerShell) ensurePrepareStageHostnameMessage(
w ShellWriter,
buildStage common.BuildStage,
info common.ShellScriptInfo,
) {
if buildStage == common.BuildStagePrepare {
if info.Build.Hostname != "" {
w.Line(
fmt.Sprintf(
`echo "Running on $([Environment]::MachineName) via %s..."`,
psQuoteVariable(info.Build.Hostname),
),
)
} else {
w.Line(`echo "Running on $([Environment]::MachineName)..."`)
}
}
}
func (b *PowerShell) IsDefault() bool {
return false
}
func init() {
eol := "\r\n"
if runtime.GOOS != OSWindows {
eol = "\n"
}
common.RegisterShell(&PowerShell{Shell: SNPwsh, EOL: eol})
common.RegisterShell(&PowerShell{Shell: SNPowershell, EOL: "\r\n"})
}
package shells
import (
"bufio"
"bytes"
"fmt"
"io"
"strings"
"gitlab.com/gitlab-org/gitlab-runner/common"
)
// pwshTrapShellScript is used to wrap a shell script in a trap that makes sure the script always exits
// with exit code of 0. This can be useful in container environments where exiting with an exit code different from 0
// would kill the container.
// At the same time it writes to a file the actual exit code of the script as well as the filename
// At the same time it writes the actual exit code of the script as well as
// the filename of the script (as json) to a file.
// With powershell $? returns True if the last command was successful so the exit_code is set to 0 in that case
const pwshTrapShellScript = `
function runner_script_trap() {
$lastExit = $?
$code = 1
If($lastExit -eq "True"){ $code = 0 }
$log_file=%q
$out_json= '{"command_exit_code": ' + $code + ', "script": "' + $MyInvocation.MyCommand.Name + '"}'
# Make sure the command status will always be printed on a new line
if ( $((Get-Content -Path $log_file | Measure-Object -Line).Lines) -gt 0 )
{
Add-Content $log_file "$out_json"
}
else
{
Add-Content $log_file ""
Add-Content $log_file "$out_json"
}
}
trap {runner_script_trap}
`
type PwshTrapShellWriter struct {
*PsWriter
logFile string
}
func (b *PwshTrapShellWriter) Finish(trace bool) string {
var buffer bytes.Buffer
w := bufio.NewWriter(&buffer)
b.writeShebang(w)
b.writeTrap(w)
b.writeTrace(w, trace)
b.writeScript(w)
_ = w.Flush()
return buffer.String()
}
func (b *PwshTrapShellWriter) writeTrap(w io.Writer) {
// For code readability purpose, the pwshTrapShellScript is written with \n as EOL within the script
// However when written into the generated script for a job, the \n used within the trap script is
// replaced by the shell EOL to avoid having multiple EOL within it and to keep it consistent
_, _ = fmt.Fprintf(w, strings.ReplaceAll(pwshTrapShellScript, "\n", b.EOL), b.logFile)
}
type PwshTrapShell struct {
*PowerShell
LogFile string
}
func (b *PwshTrapShell) GenerateScript(buildStage common.BuildStage, info common.ShellScriptInfo) (string, error) {
w := &PwshTrapShellWriter{
PsWriter: &PsWriter{
TemporaryPath: info.Build.TmpProjectDir(),
Shell: b.Shell,
EOL: b.EOL,
},
logFile: b.LogFile,
}
return b.generateScript(w, buildStage, info)
}
package shellstest
import (
"testing"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-runner/helpers"
"gitlab.com/gitlab-org/gitlab-runner/shells"
)
type shellWriterFactory func() shells.ShellWriter
func OnEachShell(t *testing.T, f func(t *testing.T, shell string)) {
shells := []string{"bash", "cmd", "powershell", "pwsh"}
for _, shell := range shells {
t.Run(shell, func(t *testing.T) {
helpers.SkipIntegrationTests(t, shell)
f(t, shell)
})
}
}
func OnEachShellWithWriter(t *testing.T, f func(t *testing.T, shell string, writer shells.ShellWriter)) {
writers := map[string]shellWriterFactory{
"bash": func() shells.ShellWriter {
return &shells.BashWriter{}
},
"cmd": func() shells.ShellWriter {
return &shells.CmdWriter{}
},
"powershell": func() shells.ShellWriter {
return &shells.PsWriter{Shell: "powershell", EOL: "\r\n"}
},
"pwsh": func() shells.ShellWriter {
return &shells.PsWriter{Shell: "pwsh", EOL: "\n"}
},
}
OnEachShell(t, func(t *testing.T, shell string) {
writer, ok := writers[shell]
require.True(t, ok, "Missing factory for %s", shell)
f(t, shell, writer())
})
}
package shells
import (
"encoding/json"
)
type TrapCommandExitStatus struct {
// CommandExitCode is the exit code of the last command. **REQUIRED**.
CommandExitCode *int `json:"command_exit_code"`
// Script is the script which was executed as an entrypoint for the current execution step.
// The scripts are currently named after the stage they are executed in.
// This property is **NOT REQUIRED** and may be nil in some cases.
// For example, when an error is reported by the log processor itself and not the script it was monitoring.
Script *string `json:"script"`
}
func (c *TrapCommandExitStatus) hasRequiredFields() bool {
return c != nil && c.CommandExitCode != nil
}
// TryUnmarshal tries to unmarshal a json string into its pointer receiver.
// It wil return true only if the unmarshalled struct has all of its required fields be non-nil.
// It's safe to use the struct only if this method returns true.
func (c *TrapCommandExitStatus) TryUnmarshal(line string) bool {
err := json.Unmarshal([]byte(line), c)
if err != nil {
return false
}
if !c.hasRequiredFields() {
return false
}
return true
}