THIS IS A TEST INSTANCE ONLY! REPOSITORIES CAN BE DELETED AT ANY TIME!

Browse Source

feat(edge): introduce support for Edge agent (#3031)

* feat(edge): fix webconsole and agent deployment command

* feat(edge): display agent features when connected to IoT endpoint

* feat(edge): add -e CAP_HOST_MANAGEMENT=1 to agent command

* feat(edge): add -v /:/host and --name portainer_agent_iot to agent command

* style(endpoint-creation): refactor IoT agent to Edge agent

* refactor(api): rename AgentIoTEnvironment to AgentEdgeEnvironment

* refactor(api): rename AgentIoTEnvironment to AgentEdgeEnvironment

* feat(endpoint-creation): update Edge agent deployment instructions

* feat(edge): wip edge

* feat(edge): refactor key creation

* feat(edge): update deployment instructions

* feat(home): update Edge agent endpoint item

* feat(edge): support dynamic ports

* feat(edge): support sleep/wake and snapshots

* feat(edge): support offline mode

* feat(edge): host job support for Edge endpoints

* feat(edge): introduce STANDBY state

* feat(edge): update Edge agent deployment command

* feat(edge): introduce EDGE_ID support

* feat(edge): update default inactivity interval to 5min

* feat(edge): reload Edge schedules after restart

* fix(edge): fix execution of endpoint job against an Edge endpoint

* fix(edge): fix minor issues with scheduling UI/UX

* feat(edge): introduce EdgeSchedule version management

* feat(edge): switch back to REQUIRED state from ACTIVE on error

* refactor(edge): remove comment

* feat(edge): updated tunnel status management

* feat(edge): fix flickering UI when accessing Edge endpoint from home view

* feat(edge): remove STANDBY status

* fix(edge): fix an issue with console and Swarm endpoint

* fix(edge): fix an issue with stack deployment

* fix(edge): reset timer when applying active status

* feat(edge): add background ping for Edge endpoints

* fix(edge): fix infinite loading loop after Edge endpoint connection failure

* fix(home): fix an issue with merge

* feat(api): remove SnapshotRaw from EndpointList response

* feat(api): add pagination for EndpointList operation

* feat(api): rename last_id query parameter to start

* feat(api): implement filter for EndpointList operation

* fix(edge): prevent a pointer issue after removing an active Edge endpoint

* feat(home): front - endpoint backend pagination (#2990)

* feat(home): endpoint pagination with backend

* feat(api): remove default limit value

* fix(endpoints): fix a minor issue with column span

* fix(endpointgroup-create): fix an issue with endpoint group creation

* feat(app): minor loading optimizations

* refactor(api): small refactor of EndpointList operation

* fix(home): fix minor loading text display issue

* refactor(api): document bolt services functions

* feat(home): minor optimization

* fix(api): replace seek with index scanning for EndpointPaginated

* fix(api): fix invalid starting index issue

* fix(api): first implementation of working filter

* fix(home): endpoints list keeps backend pagination when it needs to

* fix(api): endpoint pagination doesn't drop the first item on pages >=2 anymore

* fix(home): UI flickering on page/filter load/change

* feat(auth): login spinner

* feat(api): support searching in associated endpoint group data

* refactor(api): remove unused API endpoint

* refactor(api): remove comment

* refactor(api): refactor proxy manager

* feat(api): declare EndpointList params as optional

* feat(api): support groupID filter on endpoints route

* feat(api): add new API operations endpointGroupAddEndpoint and endpointGroupDeleteEndpoint

* feat(edge): new icon for Edge agent endpoint

* fix(edge): fix missing exec quick action

* fix(edge): add loading indicator when connecting to Edge endpoint

* feat(edge): disable service webhooks for Edge endpoints

* feat(endpoints): backend pagination for endpoints view (#3004)

* feat(edge): dynamic loading for stack migration feature

* feat(edge): wordwrap edge key

* feat(endpoint-groups): backend pagination support for create and edit

* feat(endpoint-groups): debounce on filter for create/edit views

* feat(endpoint-groups): filter assigned on create view

* (endpoint-groups): unassigned endpoints edit view

* refactor(endpoint-groups): code clean

* feat(endpoint-groups): remove message for Unassigned group

* refactor(websocket): minor refactor associated to Edge agent

* feat(endpoint-group): enable backend pagination (#3017)

* feat(api): support groupID filter on endpoints route

* feat(api): add new API operations endpointGroupAddEndpoint and endpointGroupDeleteEndpoint

* feat(endpoint-groups): backend pagination support for create and edit

* feat(endpoint-groups): debounce on filter for create/edit views

* feat(endpoint-groups): filter assigned on create view

* (endpoint-groups): unassigned endpoints edit view

* refactor(endpoint-groups): code clean

* feat(endpoint-groups): remove message for Unassigned group

* refactor(api): endpoint group endpoint association refactor

* refactor(api): rename files and remove comments

* refactor(api): remove usage of utils

* refactor(api): optional parameters

* Merge branch 'feat-endpoint-backend-pagination' into edge

# Conflicts:
#	api/bolt/endpoint/endpoint.go
#	api/http/handler/endpointgroups/endpointgroup_update.go
#	api/http/handler/endpointgroups/handler.go
#	api/http/handler/endpoints/endpoint_list.go
#	app/portainer/services/api/endpointService.js

* fix(api): fix default tunnel server credentials

* feat(api): update endpointListOperation behavior and parameters

* fix(api): fix interface declaration

* feat(edge): support configurable Edge agent checkin interval

* feat(edge): support dynamic tunnel credentials

* feat(edge): update Edge agent deployment commands

* style(edge): update Edge agent settings text

* refactor(edge): remove unused credentials management methods

* feat(edge): associate a remote addr to tunnel credentials

* style(edge): update Edge endpoint icon

* feat(edge): support encrypted tunnel credentials

* fix(edge): fix invalid pointer cast

* feat(bolt): decode endpoints with jsoniter

* feat(edge): persist reverse tunnel keyseed

* refactor(edge): minor refactor

* feat(edge): update chisel library usage

* refactor(endpoint): use controller function

* feat(api): database migration to DBVersion 19

* refactor(api): refactor AddSchedule function

* refactor(schedules): remove comment

* refactor(api): remove comment

* refactor(api): remove comment

* feat(api): tunnel manager now only manage Edge endpoints

* refactor(api): clean-up and clarification of the Edge service

* refactor(api): clean-up and clarification of the Edge service

* fix(api): fix an issue with Edge agent snapshots

* refactor(api): add missing comments

* refactor(api): update constant description

* style(home): remove loading text on error

* feat(endpoint): remove 15s timeout for ping request

* style(home): display information about associated Edge endpoints

* feat(home): redirect to endpoint details on click on unassociated Edge endpoint

* feat(settings): remove 60s Edge poll frequency option
tags/1.22.0^2
Anthony Lapenna GitHub 7 months ago
parent
commit
12a512f01f
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
86 changed files with 1567 additions and 224 deletions
  1. +9
    -0
      api/bolt/datastore.go
  2. +1
    -1
      api/bolt/endpoint/endpoint.go
  3. +10
    -0
      api/bolt/internal/json.go
  4. +16
    -0
      api/bolt/migrator/migrate_dbversion18.go
  5. +8
    -0
      api/bolt/migrator/migrator.go
  6. +48
    -0
      api/bolt/tunnelserver/tunnelserver.go
  7. +24
    -0
      api/chisel/key.go
  8. +47
    -0
      api/chisel/schedules.go
  9. +191
    -0
      api/chisel/service.go
  10. +144
    -0
      api/chisel/tunnel.go
  11. +2
    -0
      api/cli/cli.go
  12. +19
    -17
      api/cli/defaults.go
  13. +19
    -17
      api/cli/defaults_windows.go
  14. +30
    -13
      api/cmd/portainer/main.go
  15. +1
    -1
      api/cron/job_snapshot.go
  16. +3
    -1
      api/crypto/ecdsa.go
  17. +0
    -0
      api/crypto/hash.go
  18. +0
    -10
      api/crypto/md5.go
  19. +30
    -3
      api/docker/client.go
  20. +25
    -15
      api/exec/swarm_stack.go
  21. +5
    -3
      api/http/handler/endpointproxy/handler.go
  22. +1
    -1
      api/http/handler/endpointproxy/proxy_azure.go
  23. +27
    -2
      api/http/handler/endpointproxy/proxy_docker.go
  24. +52
    -1
      api/http/handler/endpoints/endpoint_create.go
  25. +1
    -1
      api/http/handler/endpoints/endpoint_delete.go
  26. +77
    -0
      api/http/handler/endpoints/endpoint_status_inspect.go
  27. +5
    -0
      api/http/handler/endpoints/handler.go
  28. +2
    -2
      api/http/handler/motd/motd.go
  29. +7
    -6
      api/http/handler/schedules/handler.go
  30. +43
    -1
      api/http/handler/schedules/schedule_create.go
  31. +2
    -0
      api/http/handler/schedules/schedule_delete.go
  32. +19
    -0
      api/http/handler/schedules/schedule_tasks.go
  33. +50
    -1
      api/http/handler/schedules/schedule_update.go
  34. +5
    -0
      api/http/handler/settings/settings_update.go
  35. +4
    -2
      api/http/handler/websocket/attach.go
  36. +4
    -2
      api/http/handler/websocket/exec.go
  37. +5
    -4
      api/http/handler/websocket/handler.go
  38. +26
    -3
      api/http/handler/websocket/proxy.go
  39. +15
    -1
      api/http/proxy/docker_transport.go
  40. +17
    -8
      api/http/proxy/factory.go
  41. +4
    -2
      api/http/proxy/factory_local.go
  42. +5
    -4
      api/http/proxy/factory_local_windows.go
  43. +35
    -18
      api/http/proxy/manager.go
  44. +8
    -0
      api/http/server.go
  45. +17
    -7
      api/libcompose/compose_stack.go
  46. +65
    -2
      api/portainer.go
  47. +14
    -2
      app/app.js
  48. +1
    -0
      app/docker/components/datatables/service-tasks-datatable/serviceTasksDatatableController.js
  49. +1
    -1
      app/docker/helpers/infoHelper.js
  50. +2
    -2
      app/docker/rest/container.js
  51. +3
    -3
      app/docker/rest/image.js
  52. +2
    -2
      app/docker/rest/network.js
  53. +2
    -2
      app/docker/rest/system.js
  54. +1
    -1
      app/docker/rest/systemEndpoint.js
  55. +2
    -2
      app/docker/rest/volume.js
  56. +1
    -1
      app/docker/services/systemService.js
  57. +1
    -1
      app/docker/views/services/edit/service.html
  58. +3
    -3
      app/extensions/rbac/components/access-viewer/accessViewerController.js
  59. +7
    -3
      app/portainer/components/datatables/schedule-tasks-datatable/scheduleTasksDatatable.html
  60. +2
    -1
      app/portainer/components/datatables/schedule-tasks-datatable/scheduleTasksDatatable.js
  61. +14
    -4
      app/portainer/components/endpoint-list/endpoint-item/endpointItem.html
  62. +0
    -1
      app/portainer/components/endpoint-list/endpoint-list-controller.js
  63. +2
    -2
      app/portainer/components/forms/schedule-form/schedule-form.js
  64. +31
    -3
      app/portainer/components/forms/schedule-form/scheduleForm.html
  65. +4
    -0
      app/portainer/filters/filters.js
  66. +12
    -4
      app/portainer/models/schedule.js
  67. +1
    -0
      app/portainer/models/settings.js
  68. +4
    -3
      app/portainer/rest/endpoint.js
  69. +6
    -1
      app/portainer/services/api/endpointService.js
  70. +1
    -5
      app/portainer/services/api/groupService.js
  71. +1
    -0
      app/portainer/services/stateManager.js
  72. +24
    -3
      app/portainer/views/endpoints/create/createEndpointController.js
  73. +46
    -2
      app/portainer/views/endpoints/create/createendpoint.html
  74. +101
    -3
      app/portainer/views/endpoints/edit/endpoint.html
  75. +39
    -4
      app/portainer/views/endpoints/edit/endpointController.js
  76. +6
    -1
      app/portainer/views/home/home.html
  77. +36
    -5
      app/portainer/views/home/homeController.js
  78. +1
    -1
      app/portainer/views/schedules/create/createScheduleController.js
  79. +1
    -0
      app/portainer/views/schedules/edit/schedule.html
  80. +23
    -2
      app/portainer/views/schedules/edit/scheduleController.js
  81. +17
    -1
      app/portainer/views/settings/settings.html
  82. +15
    -1
      app/portainer/views/settings/settingsController.js
  83. +8
    -2
      app/portainer/views/stacks/edit/stackController.js
  84. BIN
      assets/images/edge_endpoint.png
  85. +3
    -3
      gruntfile.js
  86. +1
    -0
      package.json

+ 9
- 0
api/bolt/datastore.go View File

@@ -5,6 +5,8 @@ import (
"path"
"time"

"github.com/portainer/portainer/api/bolt/tunnelserver"

"github.com/boltdb/bolt"
"github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/bolt/dockerhub"
@@ -51,6 +53,7 @@ type Store struct {
TeamMembershipService *teammembership.Service
TeamService *team.Service
TemplateService *template.Service
TunnelServerService *tunnelserver.Service
UserService *user.Service
VersionService *version.Service
WebhookService *webhook.Service
@@ -220,6 +223,12 @@ func (store *Store) initServices() error {
}
store.TemplateService = templateService

tunnelServerService, err := tunnelserver.NewService(store.db)
if err != nil {
return err
}
store.TunnelServerService = tunnelServerService

userService, err := user.NewService(store.db)
if err != nil {
return err


+ 1
- 1
api/bolt/endpoint/endpoint.go View File

@@ -63,7 +63,7 @@ func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
var endpoint portainer.Endpoint
err := internal.UnmarshalObject(v, &endpoint)
err := internal.UnmarshalObjectWithJsoniter(v, &endpoint)
if err != nil {
return err
}


+ 10
- 0
api/bolt/internal/json.go View File

@@ -2,6 +2,8 @@ package internal

import (
"encoding/json"

jsoniter "github.com/json-iterator/go"
)

// MarshalObject encodes an object to binary format
@@ -13,3 +15,11 @@ func MarshalObject(object interface{}) ([]byte, error) {
func UnmarshalObject(data []byte, object interface{}) error {
return json.Unmarshal(data, object)
}

// UnmarshalObjectWithJsoniter decodes an object from binary data
// using the jsoniter library. It is mainly used to accelerate endpoint
// decoding at the moment.
func UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary
return jsoni.Unmarshal(data, &object)
}

+ 16
- 0
api/bolt/migrator/migrate_dbversion18.go View File

@@ -0,0 +1,16 @@
package migrator

import portainer "github.com/portainer/portainer/api"

func (m *Migrator) updateSettingsToDBVersion19() error {
legacySettings, err := m.settingsService.Settings()
if err != nil {
return err
}

if legacySettings.EdgeAgentCheckinInterval == 0 {
legacySettings.EdgeAgentCheckinInterval = portainer.DefaultEdgeAgentCheckinIntervalInSeconds
}

return m.settingsService.UpdateSettings(legacySettings)
}

+ 8
- 0
api/bolt/migrator/migrator.go View File

@@ -249,5 +249,13 @@ func (m *Migrator) Migrate() error {
}
}

// Portainer 1.21.1
if m.currentDBVersion < 19 {
err := m.updateSettingsToDBVersion19()
if err != nil {
return err
}
}

return m.versionService.StoreDBVersion(portainer.DBVersion)
}

+ 48
- 0
api/bolt/tunnelserver/tunnelserver.go View File

@@ -0,0 +1,48 @@
package tunnelserver

import (
"github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/bolt/internal"

"github.com/boltdb/bolt"
)

const (
// BucketName represents the name of the bucket where this service stores data.
BucketName = "tunnel_server"
infoKey = "INFO"
)

// Service represents a service for managing endpoint data.
type Service struct {
db *bolt.DB
}

// NewService creates a new instance of a service.
func NewService(db *bolt.DB) (*Service, error) {
err := internal.CreateBucket(db, BucketName)
if err != nil {
return nil, err
}

return &Service{
db: db,
}, nil
}

// Info retrieve the TunnelServerInfo object.
func (service *Service) Info() (*portainer.TunnelServerInfo, error) {
var info portainer.TunnelServerInfo

err := internal.GetObject(service.db, BucketName, []byte(infoKey), &info)
if err != nil {
return nil, err
}

return &info, nil
}

// UpdateInfo persists a TunnelServerInfo object.
func (service *Service) UpdateInfo(settings *portainer.TunnelServerInfo) error {
return internal.UpdateObject(service.db, BucketName, []byte(infoKey), settings)
}

+ 24
- 0
api/chisel/key.go View File

@@ -0,0 +1,24 @@
package chisel

import (
"encoding/base64"
"fmt"
"strconv"
"strings"
)

// GenerateEdgeKey will generate a key that can be used by an Edge agent to register with a Portainer instance.
// The key represents the following data in this particular format:
// portainer_instance_url|tunnel_server_addr|tunnel_server_fingerprint|endpoint_ID
// The key returned by this function is a base64 encoded version of the data.
func (service *Service) GenerateEdgeKey(url, host string, endpointIdentifier int) string {
keyInformation := []string{
url,
fmt.Sprintf("%s:%s", host, service.serverPort),
service.serverFingerprint,
strconv.Itoa(endpointIdentifier),
}

key := strings.Join(keyInformation, "|")
return base64.RawStdEncoding.EncodeToString([]byte(key))
}

+ 47
- 0
api/chisel/schedules.go View File

@@ -0,0 +1,47 @@
package chisel

import (
"strconv"

portainer "github.com/portainer/portainer/api"
)

// AddSchedule register a schedule inside the tunnel details associated to an endpoint.
func (service *Service) AddSchedule(endpointID portainer.EndpointID, schedule *portainer.EdgeSchedule) {
tunnel := service.GetTunnelDetails(endpointID)

existingScheduleIndex := -1
for idx, existingSchedule := range tunnel.Schedules {
if existingSchedule.ID == schedule.ID {
existingScheduleIndex = idx
break
}
}

if existingScheduleIndex == -1 {
tunnel.Schedules = append(tunnel.Schedules, *schedule)
} else {
tunnel.Schedules[existingScheduleIndex] = *schedule
}

key := strconv.Itoa(int(endpointID))
service.tunnelDetailsMap.Set(key, tunnel)
}

// RemoveSchedule will remove the specified schedule from each tunnel it was registered with.
func (service *Service) RemoveSchedule(scheduleID portainer.ScheduleID) {
for item := range service.tunnelDetailsMap.IterBuffered() {
tunnelDetails := item.Val.(*portainer.TunnelDetails)

updatedSchedules := make([]portainer.EdgeSchedule, 0)
for _, schedule := range tunnelDetails.Schedules {
if schedule.ID == scheduleID {
continue
}
updatedSchedules = append(updatedSchedules, schedule)
}

tunnelDetails.Schedules = updatedSchedules
service.tunnelDetailsMap.Set(item.Key, tunnelDetails)
}
}

+ 191
- 0
api/chisel/service.go View File

@@ -0,0 +1,191 @@
package chisel

import (
"fmt"
"log"
"strconv"
"time"

"github.com/dchest/uniuri"

cmap "github.com/orcaman/concurrent-map"

chserver "github.com/jpillora/chisel/server"
portainer "github.com/portainer/portainer/api"
)

const (
tunnelCleanupInterval = 10 * time.Second
requiredTimeout = 15 * time.Second
activeTimeout = 4*time.Minute + 30*time.Second
)

// Service represents a service to manage the state of multiple reverse tunnels.
// It is used to start a reverse tunnel server and to manage the connection status of each tunnel
// connected to the tunnel server.
type Service struct {
serverFingerprint string
serverPort string
tunnelDetailsMap cmap.ConcurrentMap
endpointService portainer.EndpointService
tunnelServerService portainer.TunnelServerService
snapshotter portainer.Snapshotter
chiselServer *chserver.Server
}

// NewService returns a pointer to a new instance of Service
func NewService(endpointService portainer.EndpointService, tunnelServerService portainer.TunnelServerService) *Service {
return &Service{
tunnelDetailsMap: cmap.New(),
endpointService: endpointService,
tunnelServerService: tunnelServerService,
}
}

// StartTunnelServer starts a tunnel server on the specified addr and port.
// It uses a seed to generate a new private/public key pair. If the seed cannot
// be found inside the database, it will generate a new one randomly and persist it.
// It starts the tunnel status verification process in the background.
// The snapshotter is used in the tunnel status verification process.
func (service *Service) StartTunnelServer(addr, port string, snapshotter portainer.Snapshotter) error {
keySeed, err := service.retrievePrivateKeySeed()
if err != nil {
return err
}

config := &chserver.Config{
Reverse: true,
KeySeed: keySeed,
}

chiselServer, err := chserver.NewServer(config)
if err != nil {
return err
}

service.serverFingerprint = chiselServer.GetFingerprint()
service.serverPort = port

err = chiselServer.Start(addr, port)
if err != nil {
return err
}
service.chiselServer = chiselServer

// TODO: work-around Chisel default behavior.
// By default, Chisel will allow anyone to connect if no user exists.
username, password := generateRandomCredentials()
err = service.chiselServer.AddUser(username, password, "127.0.0.1")
if err != nil {
return err
}

service.snapshotter = snapshotter
go service.startTunnelVerificationLoop()

return nil
}

func (service *Service) retrievePrivateKeySeed() (string, error) {
var serverInfo *portainer.TunnelServerInfo

serverInfo, err := service.tunnelServerService.Info()
if err == portainer.ErrObjectNotFound {
keySeed := uniuri.NewLen(16)

serverInfo = &portainer.TunnelServerInfo{
PrivateKeySeed: keySeed,
}

err := service.tunnelServerService.UpdateInfo(serverInfo)
if err != nil {
return "", err
}
} else if err != nil {
return "", err
}

return serverInfo.PrivateKeySeed, nil
}

func (service *Service) startTunnelVerificationLoop() {
log.Printf("[DEBUG] [chisel, monitoring] [check_interval_seconds: %f] [message: starting tunnel management process]", tunnelCleanupInterval.Seconds())
ticker := time.NewTicker(tunnelCleanupInterval)
stopSignal := make(chan struct{})

for {
select {
case <-ticker.C:
service.checkTunnels()
case <-stopSignal:
ticker.Stop()
return
}
}
}

func (service *Service) checkTunnels() {
for item := range service.tunnelDetailsMap.IterBuffered() {
tunnel := item.Val.(*portainer.TunnelDetails)

if tunnel.LastActivity.IsZero() || tunnel.Status == portainer.EdgeAgentIdle {
continue
}

elapsed := time.Since(tunnel.LastActivity)
log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %s] [status: %s] [status_time_seconds: %f] [message: endpoint tunnel monitoring]", item.Key, tunnel.Status, elapsed.Seconds())

if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed.Seconds() < requiredTimeout.Seconds() {
continue
} else if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed.Seconds() > requiredTimeout.Seconds() {
log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %s] [status: %s] [status_time_seconds: %f] [timeout_seconds: %f] [message: REQUIRED state timeout exceeded]", item.Key, tunnel.Status, elapsed.Seconds(), requiredTimeout.Seconds())
}

if tunnel.Status == portainer.EdgeAgentActive && elapsed.Seconds() < activeTimeout.Seconds() {
continue
} else if tunnel.Status == portainer.EdgeAgentActive && elapsed.Seconds() > activeTimeout.Seconds() {
log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %s] [status: %s] [status_time_seconds: %f] [timeout_seconds: %f] [message: ACTIVE state timeout exceeded]", item.Key, tunnel.Status, elapsed.Seconds(), activeTimeout.Seconds())

endpointID, err := strconv.Atoi(item.Key)
if err != nil {
log.Printf("[ERROR] [chisel,snapshot,conversion] Invalid endpoint identifier (id: %s): %s", item.Key, err)
}

err = service.snapshotEnvironment(portainer.EndpointID(endpointID), tunnel.Port)
if err != nil {
log.Printf("[ERROR] [snapshot] Unable to snapshot Edge endpoint (id: %s): %s", item.Key, err)
}
}

if len(tunnel.Schedules) > 0 {
endpointID, err := strconv.Atoi(item.Key)
if err != nil {
log.Printf("[ERROR] [chisel,conversion] Invalid endpoint identifier (id: %s): %s", item.Key, err)
continue
}

service.SetTunnelStatusToIdle(portainer.EndpointID(endpointID))
} else {
service.tunnelDetailsMap.Remove(item.Key)
}

}
}

func (service *Service) snapshotEnvironment(endpointID portainer.EndpointID, tunnelPort int) error {
endpoint, err := service.endpointService.Endpoint(portainer.EndpointID(endpointID))
if err != nil {
return err
}

endpointURL := endpoint.URL
endpoint.URL = fmt.Sprintf("tcp://localhost:%d", tunnelPort)
snapshot, err := service.snapshotter.CreateSnapshot(endpoint)
if err != nil {
return err
}

endpoint.Snapshots = []portainer.Snapshot{*snapshot}
endpoint.URL = endpointURL
return service.endpointService.UpdateEndpoint(endpoint.ID, endpoint)
}

+ 144
- 0
api/chisel/tunnel.go View File

@@ -0,0 +1,144 @@
package chisel

import (
"encoding/base64"
"fmt"
"math/rand"
"strconv"
"strings"
"time"

"github.com/portainer/libcrypto"

"github.com/dchest/uniuri"
portainer "github.com/portainer/portainer/api"
)

const (
minAvailablePort = 49152
maxAvailablePort = 65535
)

// getUnusedPort is used to generate an unused random port in the dynamic port range.
// Dynamic ports (also called private ports) are 49152 to 65535.
func (service *Service) getUnusedPort() int {
port := randomInt(minAvailablePort, maxAvailablePort)

for item := range service.tunnelDetailsMap.IterBuffered() {
tunnel := item.Val.(*portainer.TunnelDetails)
if tunnel.Port == port {
return service.getUnusedPort()
}
}

return port
}

func randomInt(min, max int) int {
return min + rand.Intn(max-min)
}

// GetTunnelDetails returns information about the tunnel associated to an endpoint.
func (service *Service) GetTunnelDetails(endpointID portainer.EndpointID) *portainer.TunnelDetails {
key := strconv.Itoa(int(endpointID))

if item, ok := service.tunnelDetailsMap.Get(key); ok {
tunnelDetails := item.(*portainer.TunnelDetails)
return tunnelDetails
}

schedules := make([]portainer.EdgeSchedule, 0)
return &portainer.TunnelDetails{
Status: portainer.EdgeAgentIdle,
Port: 0,
Schedules: schedules,
Credentials: "",
}
}

// SetTunnelStatusToActive update the status of the tunnel associated to the specified endpoint.
// It sets the status to ACTIVE.
func (service *Service) SetTunnelStatusToActive(endpointID portainer.EndpointID) {
tunnel := service.GetTunnelDetails(endpointID)
tunnel.Status = portainer.EdgeAgentActive
tunnel.Credentials = ""
tunnel.LastActivity = time.Now()

key := strconv.Itoa(int(endpointID))
service.tunnelDetailsMap.Set(key, tunnel)
}

// SetTunnelStatusToIdle update the status of the tunnel associated to the specified endpoint.
// It sets the status to IDLE.
// It removes any existing credentials associated to the tunnel.
func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
tunnel := service.GetTunnelDetails(endpointID)

tunnel.Status = portainer.EdgeAgentIdle
tunnel.Port = 0
tunnel.LastActivity = time.Now()

credentials := tunnel.Credentials
if credentials != "" {
tunnel.Credentials = ""
service.chiselServer.DeleteUser(strings.Split(credentials, ":")[0])
}

key := strconv.Itoa(int(endpointID))
service.tunnelDetailsMap.Set(key, tunnel)
}

// SetTunnelStatusToRequired update the status of the tunnel associated to the specified endpoint.
// It sets the status to REQUIRED.
// If no port is currently associated to the tunnel, it will associate a random unused port to the tunnel
// and generate temporary credentials that can be used to establish a reverse tunnel on that port.
// Credentials are encrypted using the Edge ID associated to the endpoint.
func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointID) error {
tunnel := service.GetTunnelDetails(endpointID)

if tunnel.Port == 0 {
endpoint, err := service.endpointService.Endpoint(endpointID)
if err != nil {
return err
}

tunnel.Status = portainer.EdgeAgentManagementRequired
tunnel.Port = service.getUnusedPort()
tunnel.LastActivity = time.Now()

username, password := generateRandomCredentials()
authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tunnel.Port)
err = service.chiselServer.AddUser(username, password, authorizedRemote)
if err != nil {
return err
}

credentials, err := encryptCredentials(username, password, endpoint.EdgeID)
if err != nil {
return err
}
tunnel.Credentials = credentials

key := strconv.Itoa(int(endpointID))
service.tunnelDetailsMap.Set(key, tunnel)
}

return nil
}

func generateRandomCredentials() (string, string) {
username := uniuri.NewLen(8)
password := uniuri.NewLen(8)
return username, password
}

func encryptCredentials(username, password, key string) (string, error) {
credentials := fmt.Sprintf("%s:%s", username, password)

encryptedCredentials, err := libcrypto.Encrypt([]byte(credentials), []byte(key))
if err != nil {
return "", err
}

return base64.RawStdEncoding.EncodeToString(encryptedCredentials), nil
}

+ 2
- 0
api/cli/cli.go View File

@@ -33,6 +33,8 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {

flags := &portainer.CLIFlags{
Addr: kingpin.Flag("bind", "Address and port to serve Portainer").Default(defaultBindAddress).Short('p').String(),
TunnelAddr: kingpin.Flag("tunnel-addr", "Address to serve the tunnel server").Default(defaultTunnelServerAddress).String(),
TunnelPort: kingpin.Flag("tunnel-port", "Port to serve the tunnel server").Default(defaultTunnelServerPort).String(),
Assets: kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(),
Data: kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(),
EndpointURL: kingpin.Flag("host", "Endpoint URL").Short('H').String(),


+ 19
- 17
api/cli/defaults.go View File

@@ -3,21 +3,23 @@
package cli

const (
defaultBindAddress = ":9000"
defaultDataDirectory = "/data"
defaultAssetsDirectory = "./"
defaultNoAuth = "false"
defaultNoAnalytics = "false"
defaultTLS = "false"
defaultTLSSkipVerify = "false"
defaultTLSCACertPath = "/certs/ca.pem"
defaultTLSCertPath = "/certs/cert.pem"
defaultTLSKeyPath = "/certs/key.pem"
defaultSSL = "false"
defaultSSLCertPath = "/certs/portainer.crt"
defaultSSLKeyPath = "/certs/portainer.key"
defaultSyncInterval = "60s"
defaultSnapshot = "true"
defaultSnapshotInterval = "5m"
defaultTemplateFile = "/templates.json"
defaultBindAddress = ":9000"
defaultTunnelServerAddress = "0.0.0.0"
defaultTunnelServerPort = "8000"
defaultDataDirectory = "/data"
defaultAssetsDirectory = "./"
defaultNoAuth = "false"
defaultNoAnalytics = "false"
defaultTLS = "false"
defaultTLSSkipVerify = "false"
defaultTLSCACertPath = "/certs/ca.pem"
defaultTLSCertPath = "/certs/cert.pem"
defaultTLSKeyPath = "/certs/key.pem"
defaultSSL = "false"
defaultSSLCertPath = "/certs/portainer.crt"
defaultSSLKeyPath = "/certs/portainer.key"
defaultSyncInterval = "60s"
defaultSnapshot = "true"
defaultSnapshotInterval = "5m"
defaultTemplateFile = "/templates.json"
)

+ 19
- 17
api/cli/defaults_windows.go View File

@@ -1,21 +1,23 @@
package cli

const (
defaultBindAddress = ":9000"
defaultDataDirectory = "C:\\data"
defaultAssetsDirectory = "./"
defaultNoAuth = "false"
defaultNoAnalytics = "false"
defaultTLS = "false"
defaultTLSSkipVerify = "false"
defaultTLSCACertPath = "C:\\certs\\ca.pem"
defaultTLSCertPath = "C:\\certs\\cert.pem"
defaultTLSKeyPath = "C:\\certs\\key.pem"
defaultSSL = "false"
defaultSSLCertPath = "C:\\certs\\portainer.crt"
defaultSSLKeyPath = "C:\\certs\\portainer.key"
defaultSyncInterval = "60s"
defaultSnapshot = "true"
defaultSnapshotInterval = "5m"
defaultTemplateFile = "/templates.json"
defaultBindAddress = ":9000"
defaultTunnelServerAddress = "0.0.0.0"
defaultTunnelServerPort = "8000"
defaultDataDirectory = "C:\\data"
defaultAssetsDirectory = "./"
defaultNoAuth = "false"
defaultNoAnalytics = "false"
defaultTLS = "false"
defaultTLSSkipVerify = "false"
defaultTLSCACertPath = "C:\\certs\\ca.pem"
defaultTLSCertPath = "C:\\certs\\cert.pem"
defaultTLSKeyPath = "C:\\certs\\key.pem"
defaultSSL = "false"
defaultSSLCertPath = "C:\\certs\\portainer.crt"
defaultSSLKeyPath = "C:\\certs\\portainer.key"
defaultSyncInterval = "60s"
defaultSnapshot = "true"
defaultSnapshotInterval = "5m"
defaultTemplateFile = "/templates.json"
)

+ 30
- 13
api/cmd/portainer/main.go View File

@@ -2,10 +2,13 @@ package main

import (
"encoding/json"
"log"
"os"
"strings"
"time"

"github.com/portainer/portainer/api/chisel"

"github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/bolt"
"github.com/portainer/portainer/api/cli"
@@ -20,8 +23,6 @@ import (
"github.com/portainer/portainer/api/jwt"
"github.com/portainer/portainer/api/ldap"
"github.com/portainer/portainer/api/libcompose"

"log"
)

func initCLI() *portainer.CLIFlags {
@@ -69,12 +70,12 @@ func initStore(dataStorePath string, fileService portainer.FileService) *bolt.St
return store
}

func initComposeStackManager(dataStorePath string) portainer.ComposeStackManager {
return libcompose.NewComposeStackManager(dataStorePath)
func initComposeStackManager(dataStorePath string, reverseTunnelService portainer.ReverseTunnelService) portainer.ComposeStackManager {
return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService)
}

func initSwarmStackManager(assetsPath string, dataStorePath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService) (portainer.SwarmStackManager, error) {
return exec.NewSwarmStackManager(assetsPath, dataStorePath, signatureService, fileService)
func initSwarmStackManager(assetsPath string, dataStorePath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) (portainer.SwarmStackManager, error) {
return exec.NewSwarmStackManager(assetsPath, dataStorePath, signatureService, fileService, reverseTunnelService)
}

func initJWTService(authenticationEnabled bool) portainer.JWTService {
@@ -104,8 +105,8 @@ func initGitService() portainer.GitService {
return &git.Service{}
}

func initClientFactory(signatureService portainer.DigitalSignatureService) *docker.ClientFactory {
return docker.NewClientFactory(signatureService)
func initClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *docker.ClientFactory {
return docker.NewClientFactory(signatureService, reverseTunnelService)
}

func initSnapshotter(clientFactory *docker.ClientFactory) portainer.Snapshotter {
@@ -196,7 +197,7 @@ func loadEndpointSyncSystemSchedule(jobScheduler portainer.JobScheduler, schedul
return scheduleService.CreateSchedule(endpointSyncSchedule)
}

func loadSchedulesFromDatabase(jobScheduler portainer.JobScheduler, jobService portainer.JobService, scheduleService portainer.ScheduleService, endpointService portainer.EndpointService, fileService portainer.FileService) error {
func loadSchedulesFromDatabase(jobScheduler portainer.JobScheduler, jobService portainer.JobService, scheduleService portainer.ScheduleService, endpointService portainer.EndpointService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) error {
schedules, err := scheduleService.Schedules()
if err != nil {
return err
@@ -213,6 +214,13 @@ func loadSchedulesFromDatabase(jobScheduler portainer.JobScheduler, jobService p
return err
}
}

if schedule.EdgeSchedule != nil {
for _, endpointID := range schedule.EdgeSchedule.Endpoints {
reverseTunnelService.AddSchedule(endpointID, schedule.EdgeSchedule)
}
}

}

return nil
@@ -265,6 +273,7 @@ func initSettings(settingsService portainer.SettingsService, flags *portainer.CL
AllowPrivilegedModeForRegularUsers: true,
EnableHostManagementFeatures: false,
SnapshotInterval: *flags.SnapshotInterval,
EdgeAgentCheckinInterval: portainer.DefaultEdgeAgentCheckinIntervalInSeconds,
}

if *flags.Templates != "" {
@@ -540,7 +549,9 @@ func main() {
log.Fatal(err)
}

clientFactory := initClientFactory(digitalSignatureService)
reverseTunnelService := chisel.NewService(store.EndpointService, store.TunnelServerService)

clientFactory := initClientFactory(digitalSignatureService, reverseTunnelService)

jobService := initJobService(clientFactory)

@@ -551,12 +562,12 @@ func main() {
endpointManagement = false
}

swarmStackManager, err := initSwarmStackManager(*flags.Assets, *flags.Data, digitalSignatureService, fileService)
swarmStackManager, err := initSwarmStackManager(*flags.Assets, *flags.Data, digitalSignatureService, fileService, reverseTunnelService)
if err != nil {
log.Fatal(err)
}

composeStackManager := initComposeStackManager(*flags.Data)
composeStackManager := initComposeStackManager(*flags.Data, reverseTunnelService)

err = initTemplates(store.TemplateService, fileService, *flags.Templates, *flags.TemplateFile)
if err != nil {
@@ -570,7 +581,7 @@ func main() {

jobScheduler := initJobScheduler()

err = loadSchedulesFromDatabase(jobScheduler, jobService, store.ScheduleService, store.EndpointService, fileService)
err = loadSchedulesFromDatabase(jobScheduler, jobService, store.ScheduleService, store.EndpointService, fileService, reverseTunnelService)
if err != nil {
log.Fatal(err)
}
@@ -658,7 +669,13 @@ func main() {
go terminateIfNoAdminCreated(store.UserService)
}

err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotter)
if err != nil {
log.Fatal(err)
}

var server portainer.Server = &http.Server{
ReverseTunnelService: reverseTunnelService,
Status: applicationStatus,
BindAddress: *flags.Addr,
AssetsPath: *flags.Assets,


+ 1
- 1
api/cron/job_snapshot.go View File

@@ -53,7 +53,7 @@ func (runner *SnapshotJobRunner) Run() {
}

for _, endpoint := range endpoints {
if endpoint.Type == portainer.AzureEnvironment {
if endpoint.Type == portainer.AzureEnvironment || endpoint.Type == portainer.EdgeAgentEnvironment {
continue
}



+ 3
- 1
api/crypto/ecdsa.go View File

@@ -8,6 +8,8 @@ import (
"encoding/base64"
"encoding/hex"
"math/big"

"github.com/portainer/libcrypto"
)

const (
@@ -111,7 +113,7 @@ func (service *ECDSAService) CreateSignature(message string) (string, error) {
message = service.secret
}

hash := HashFromBytes([]byte(message))
hash := libcrypto.HashFromBytes([]byte(message))

r := big.NewInt(0)
s := big.NewInt(0)


api/crypto/crypto.go → api/crypto/hash.go View File


+ 0
- 10
api/crypto/md5.go View File

@@ -1,10 +0,0 @@
package crypto

import "crypto/md5"

// HashFromBytes returns the hash of the specified data
func HashFromBytes(data []byte) []byte {
digest := md5.New()
digest.Write(data)
return digest.Sum(nil)
}

+ 30
- 3
api/docker/client.go View File

@@ -1,6 +1,7 @@
package docker

import (
"fmt"
"net/http"
"strings"
"time"
@@ -17,13 +18,15 @@ const (

// ClientFactory is used to create Docker clients
type ClientFactory struct {
signatureService portainer.DigitalSignatureService
signatureService portainer.DigitalSignatureService
reverseTunnelService portainer.ReverseTunnelService
}

// NewClientFactory returns a new instance of a ClientFactory
func NewClientFactory(signatureService portainer.DigitalSignatureService) *ClientFactory {
func NewClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *ClientFactory {
return &ClientFactory{
signatureService: signatureService,
signatureService: signatureService,
reverseTunnelService: reverseTunnelService,
}
}

@@ -35,6 +38,8 @@ func (factory *ClientFactory) CreateClient(endpoint *portainer.Endpoint, nodeNam
return nil, unsupportedEnvironmentType
} else if endpoint.Type == portainer.AgentOnDockerEnvironment {
return createAgentClient(endpoint, factory.signatureService, nodeName)
} else if endpoint.Type == portainer.EdgeAgentEnvironment {
return createEdgeClient(endpoint, factory.reverseTunnelService, nodeName)
}

if strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") {
@@ -63,6 +68,28 @@ func createTCPClient(endpoint *portainer.Endpoint) (*client.Client, error) {
)
}

func createEdgeClient(endpoint *portainer.Endpoint, reverseTunnelService portainer.ReverseTunnelService, nodeName string) (*client.Client, error) {
httpCli, err := httpClient(endpoint)
if err != nil {
return nil, err
}

headers := map[string]string{}
if nodeName != "" {
headers[portainer.PortainerAgentTargetHeader] = nodeName
}

tunnel := reverseTunnelService.GetTunnelDetails(endpoint.ID)
endpointURL := fmt.Sprintf("http://localhost:%d", tunnel.Port)

return client.NewClientWithOpts(
client.WithHost(endpointURL),
client.WithVersion(portainer.SupportedDockerAPIVersion),
client.WithHTTPClient(httpCli),
client.WithHTTPHeaders(headers),
)
}

func createAgentClient(endpoint *portainer.Endpoint, signatureService portainer.DigitalSignatureService, nodeName string) (*client.Client, error) {
httpCli, err := httpClient(endpoint)
if err != nil {


+ 25
- 15
api/exec/swarm_stack.go View File

@@ -3,6 +3,7 @@ package exec
import (
"bytes"
"encoding/json"
"fmt"
"os"
"os/exec"
"path"
@@ -13,20 +14,22 @@ import (

// SwarmStackManager represents a service for managing stacks.
type SwarmStackManager struct {
binaryPath string
dataPath string
signatureService portainer.DigitalSignatureService
fileService portainer.FileService
binaryPath string
dataPath string
signatureService portainer.DigitalSignatureService
fileService portainer.FileService
reverseTunnelService portainer.ReverseTunnelService
}

// NewSwarmStackManager initializes a new SwarmStackManager service.
// It also updates the configuration of the Docker CLI binary.
func NewSwarmStackManager(binaryPath, dataPath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService) (*SwarmStackManager, error) {
func NewSwarmStackManager(binaryPath, dataPath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) (*SwarmStackManager, error) {
manager := &SwarmStackManager{
binaryPath: binaryPath,
dataPath: dataPath,
signatureService: signatureService,
fileService: fileService,
binaryPath: binaryPath,
dataPath: dataPath,
signatureService: signatureService,
fileService: fileService,
reverseTunnelService: reverseTunnelService,
}

err := manager.updateDockerCLIConfiguration(dataPath)
@@ -39,7 +42,7 @@ func NewSwarmStackManager(binaryPath, dataPath string, signatureService portaine

// Login executes the docker login command against a list of registries (including DockerHub).
func (manager *SwarmStackManager) Login(dockerhub *portainer.DockerHub, registries []portainer.Registry, endpoint *portainer.Endpoint) {
command, args := prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
for _, registry := range registries {
if registry.Authentication {
registryArgs := append(args, "login", "--username", registry.Username, "--password", registry.Password, registry.URL)
@@ -55,7 +58,7 @@ func (manager *SwarmStackManager) Login(dockerhub *portainer.DockerHub, registri

// Logout executes the docker logout command.
func (manager *SwarmStackManager) Logout(endpoint *portainer.Endpoint) error {
command, args := prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
args = append(args, "logout")
return runCommandAndCaptureStdErr(command, args, nil, "")
}
@@ -63,7 +66,7 @@ func (manager *SwarmStackManager) Logout(endpoint *portainer.Endpoint) error {
// Deploy executes the docker stack deploy command.
func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, endpoint *portainer.Endpoint) error {
stackFilePath := path.Join(stack.ProjectPath, stack.EntryPoint)
command, args := prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)

if prune {
args = append(args, "stack", "deploy", "--prune", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name)
@@ -82,7 +85,7 @@ func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, end

// Remove executes the docker stack rm command.
func (manager *SwarmStackManager) Remove(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
command, args := prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint)
args = append(args, "stack", "rm", stack.Name)
return runCommandAndCaptureStdErr(command, args, nil, "")
}
@@ -106,7 +109,7 @@ func runCommandAndCaptureStdErr(command string, args []string, env []string, wor
return nil
}

func prepareDockerCommandAndArgs(binaryPath, dataPath string, endpoint *portainer.Endpoint) (string, []string) {
func (manager *SwarmStackManager) prepareDockerCommandAndArgs(binaryPath, dataPath string, endpoint *portainer.Endpoint) (string, []string) {
// Assume Linux as a default
command := path.Join(binaryPath, "docker")

@@ -116,7 +119,14 @@ func prepareDockerCommandAndArgs(binaryPath, dataPath string, endpoint *portaine

args := make([]string, 0)
args = append(args, "--config", dataPath)
args = append(args, "-H", endpoint.URL)

endpointURL := endpoint.URL
if endpoint.Type == portainer.EdgeAgentEnvironment {
tunnel := manager.reverseTunnelService.GetTunnelDetails(endpoint.ID)
endpointURL = fmt.Sprintf("tcp://localhost:%d", tunnel.Port)
}

args = append(args, "-H", endpointURL)

if endpoint.TLSConfig.TLS {
args = append(args, "--tls")


+ 5
- 3
api/http/handler/endpointproxy/handler.go View File

@@ -11,9 +11,11 @@ import (
// Handler is the HTTP handler used to proxy requests to external APIs.
type Handler struct {
*mux.Router
requestBouncer *security.RequestBouncer
EndpointService portainer.EndpointService
ProxyManager *proxy.Manager
requestBouncer *security.RequestBouncer
EndpointService portainer.EndpointService
SettingsService portainer.SettingsService
ProxyManager *proxy.Manager
ReverseTunnelService portainer.ReverseTunnelService
}

// NewHandler creates a handler to proxy requests to external APIs.


+ 1
- 1
api/http/handler/endpointproxy/proxy_azure.go View File

@@ -29,7 +29,7 @@ func (handler *Handler) proxyRequestsToAzureAPI(w http.ResponseWriter, r *http.R
}

var proxy http.Handler
proxy = handler.ProxyManager.GetProxy(string(endpointID))
proxy = handler.ProxyManager.GetProxy(endpoint)
if proxy == nil {
proxy, err = handler.ProxyManager.CreateAndRegisterProxy(endpoint)
if err != nil {


+ 27
- 2
api/http/handler/endpointproxy/proxy_docker.go View File

@@ -3,6 +3,7 @@ package endpointproxy
import (
"errors"
"strconv"
"time"

httperror "github.com/portainer/libhttp/error"
"github.com/portainer/libhttp/request"
@@ -24,7 +25,7 @@ func (handler *Handler) proxyRequestsToDockerAPI(w http.ResponseWriter, r *http.
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
}

if endpoint.Status == portainer.EndpointStatusDown {
if endpoint.Type != portainer.EdgeAgentEnvironment && endpoint.Status == portainer.EndpointStatusDown {
return &httperror.HandlerError{http.StatusServiceUnavailable, "Unable to query endpoint", errors.New("Endpoint is down")}
}

@@ -33,8 +34,32 @@ func (handler *Handler) proxyRequestsToDockerAPI(w http.ResponseWriter, r *http.
return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
}

if endpoint.Type == portainer.EdgeAgentEnvironment {
if endpoint.EdgeID == "" {
return &httperror.HandlerError{http.StatusInternalServerError, "No Edge agent registered with the endpoint", errors.New("No agent available")}
}

tunnel := handler.ReverseTunnelService.GetTunnelDetails(endpoint.ID)
if tunnel.Status == portainer.EdgeAgentIdle {
handler.ProxyManager.DeleteProxy(endpoint)

err := handler.ReverseTunnelService.SetTunnelStatusToRequired(endpoint.ID)
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update tunnel status", err}
}

settings, err := handler.SettingsService.Settings()
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
}

waitForAgentToConnect := time.Duration(settings.EdgeAgentCheckinInterval) * time.Second
time.Sleep(waitForAgentToConnect * 2)
}
}

var proxy http.Handler
proxy = handler.ProxyManager.GetProxy(string(endpointID))
proxy = handler.ProxyManager.GetProxy(endpoint)
if proxy == nil {
proxy, err = handler.ProxyManager.CreateAndRegisterProxy(endpoint)
if err != nil {


+ 52
- 1
api/http/handler/endpoints/endpoint_create.go View File

@@ -1,8 +1,11 @@
package endpoints

import (
"errors"
"log"
"net"
"net/http"
"net/url"
"runtime"
"strconv"

@@ -41,7 +44,7 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {

endpointType, err := request.RetrieveNumericMultiPartFormValue(r, "EndpointType", false)
if err != nil || endpointType == 0 {
return portainer.Error("Invalid endpoint type value. Value must be one of: 1 (Docker environment), 2 (Agent environment) or 3 (Azure environment)")
return portainer.Error("Invalid endpoint type value. Value must be one of: 1 (Docker environment), 2 (Agent environment), 3 (Azure environment) or 4 (Edge Agent environment)")
}
payload.EndpointType = endpointType

@@ -149,6 +152,8 @@ func (handler *Handler) endpointCreate(w http.ResponseWriter, r *http.Request) *
func (handler *Handler) createEndpoint(payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
if portainer.EndpointType(payload.EndpointType) == portainer.AzureEnvironment {
return handler.createAzureEndpoint(payload)
} else if portainer.EndpointType(payload.EndpointType) == portainer.EdgeAgentEnvironment {
return handler.createEdgeAgentEndpoint(payload)
}

if payload.TLS {
@@ -195,6 +200,52 @@ func (handler *Handler) createAzureEndpoint(payload *endpointCreatePayload) (*po
return endpoint, nil
}

func (handler *Handler) createEdgeAgentEndpoint(payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
endpointType := portainer.EdgeAgentEnvironment
endpointID := handler.EndpointService.GetNextIdentifier()

portainerURL, err := url.Parse(payload.URL)
if err != nil {
return nil, &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint URL", err}
}

portainerHost, _, err := net.SplitHostPort(portainerURL.Host)
if err != nil {
portainerHost = portainerURL.Host
}

if portainerHost == "localhost" {
return nil, &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint URL", errors.New("cannot use localhost as endpoint URL")}
}

edgeKey := handler.ReverseTunnelService.GenerateEdgeKey(payload.URL, portainerHost, endpointID)

endpoint := &portainer.Endpoint{
ID: portainer.EndpointID(endpointID),
Name: payload.Name,
URL: portainerHost,
Type: endpointType,
GroupID: portainer.EndpointGroupID(payload.GroupID),
TLSConfig: portainer.TLSConfiguration{
TLS: false,
},
AuthorizedUsers: []portainer.UserID{},
AuthorizedTeams: []portainer.TeamID{},
Extensions: []portainer.EndpointExtension{},
Tags: payload.Tags,
Status: portainer.EndpointStatusUp,
Snapshots: []portainer.Snapshot{},
EdgeKey: edgeKey,
}

err = handler.EndpointService.CreateEndpoint(endpoint)
if err != nil {
return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint inside the database", err}
}

return endpoint, nil
}

func (handler *Handler) createUnsecuredEndpoint(payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
endpointType := portainer.DockerEnvironment



+ 1
- 1
api/http/handler/endpoints/endpoint_delete.go View File

@@ -41,7 +41,7 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove endpoint from the database", err}
}

handler.ProxyManager.DeleteProxy(string(endpointID))
handler.ProxyManager.DeleteProxy(endpoint)

return response.Empty(w)
}

+ 77
- 0
api/http/handler/endpoints/endpoint_status_inspect.go View File

@@ -0,0 +1,77 @@
package endpoints

import (
"errors"
"net/http"

httperror "github.com/portainer/libhttp/error"
"github.com/portainer/libhttp/request"
"github.com/portainer/libhttp/response"
"github.com/portainer/portainer/api"
)

type endpointStatusInspectResponse struct {
Status string `json:"status"`
Port int `json:"port"`
Schedules []portainer.EdgeSchedule `json:"schedules"`
CheckinInterval int `json:"checkin"`
Credentials string `json:"credentials"`
}

// GET request on /api/endpoints/:id/status
func (handler *Handler) endpointStatusInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
}

endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
if err == portainer.ErrObjectNotFound {
return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
} else if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
}

if endpoint.Type != portainer.EdgeAgentEnvironment {
return &httperror.HandlerError{http.StatusInternalServerError, "Status unavailable for non Edge agent endpoints", errors.New("Status unavailable")}
}

edgeIdentifier := r.Header.Get(portainer.PortainerAgentEdgeIDHeader)
if edgeIdentifier == "" {
return &httperror.HandlerError{http.StatusForbidden, "Missing Edge identifier", errors.New("missing Edge identifier")}
}

if endpoint.EdgeID != "" && endpoint.EdgeID != edgeIdentifier {
return &httperror.HandlerError{http.StatusForbidden, "Invalid Edge identifier", errors.New("invalid Edge identifier")}
}

if endpoint.EdgeID == "" {
endpoint.EdgeID = edgeIdentifier

err := handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to Unable to persist endpoint changes inside the database", err}
}
}

settings, err := handler.SettingsService.Settings()
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
}

tunnel := handler.ReverseTunnelService.GetTunnelDetails(endpoint.ID)

statusResponse := endpointStatusInspectResponse{
Status: tunnel.Status,
Port: tunnel.Port,
Schedules: tunnel.Schedules,
CheckinInterval: settings.EdgeAgentCheckinInterval,
Credentials: tunnel.Credentials,
}

if tunnel.Status == portainer.EdgeAgentManagementRequired {
handler.ReverseTunnelService.SetTunnelStatusToActive(endpoint.ID)
}

return response.JSON(w, statusResponse)
}

+ 5
- 0
api/http/handler/endpoints/handler.go View File

@@ -35,6 +35,8 @@ type Handler struct {
ProxyManager *proxy.Manager
Snapshotter portainer.Snapshotter
JobService portainer.JobService
ReverseTunnelService portainer.ReverseTunnelService
SettingsService portainer.SettingsService
}

// NewHandler creates a handler to manage endpoint operations.
@@ -65,5 +67,8 @@ func NewHandler(bouncer *security.RequestBouncer, authorizeEndpointManagement bo
bouncer.AuthorizedAccess(httperror.LoggerHandler(h.endpointJob))).Methods(http.MethodPost)
h.Handle("/endpoints/{id}/snapshot",
bouncer.AuthorizedAccess(httperror.LoggerHandler(h.endpointSnapshot))).Methods(http.MethodPost)
h.Handle("/endpoints/{id}/status",
bouncer.PublicAccess(httperror.LoggerHandler(h.endpointStatusInspect))).Methods(http.MethodGet)

return h
}

+ 2
- 2
api/http/handler/motd/motd.go View File

@@ -5,9 +5,9 @@ import (
"net/http"
"strings"

"github.com/portainer/libcrypto"
"github.com/portainer/libhttp/response"
"github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/crypto"
"github.com/portainer/portainer/api/http/client"
)

@@ -42,7 +42,7 @@ func (handler *Handler) motd(w http.ResponseWriter, r *http.Request) {

message := strings.Join(data.Message, "\n")

hash := crypto.HashFromBytes([]byte(message))
hash := libcrypto.HashFromBytes([]byte(message))
resp := motdResponse{
Title: data.Title,
Message: message,


+ 7
- 6
api/http/handler/schedules/handler.go View File

@@ -12,12 +12,13 @@ import (
// Handler is the HTTP handler used to handle schedule operations.
type Handler struct {
*mux.Router
ScheduleService portainer.ScheduleService
EndpointService portainer.EndpointService
SettingsService portainer.SettingsService
FileService portainer.FileService
JobService portainer.JobService
JobScheduler portainer.JobScheduler
ScheduleService portainer.ScheduleService
EndpointService portainer.EndpointService
SettingsService portainer.SettingsService
FileService portainer.FileService
JobService portainer.JobService
JobScheduler portainer.JobScheduler
ReverseTunnelService portainer.ReverseTunnelService
}

// NewHandler creates a handler to manage schedule operations.


+ 43
- 1
api/http/handler/schedules/schedule_create.go View File

@@ -1,9 +1,11 @@
package schedules

import (
"encoding/base64"
"errors"
"net/http"
"strconv"
"strings"
"time"

"github.com/asaskevich/govalidator"
@@ -113,7 +115,7 @@ func (payload *scheduleCreateFromFileContentPayload) Validate(r *http.Request) e
return nil
}

// POST /api/schedules?method=file/string
// POST /api/schedules?method=file|string
func (handler *Handler) scheduleCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
settings, err := handler.SettingsService.Settings()
if err != nil {
@@ -219,6 +221,46 @@ func (handler *Handler) createScheduleObjectFromFileContentPayload(payload *sche
}

func (handler *Handler) addAndPersistSchedule(schedule *portainer.Schedule, file []byte) error {
nonEdgeEndpointIDs := make([]portainer.EndpointID, 0)
edgeEndpointIDs := make([]portainer.EndpointID, 0)

edgeCronExpression := strings.Split(schedule.CronExpression, " ")
if len(edgeCronExpression) == 6 {
edgeCronExpression = edgeCronExpression[1:]
}

for _, ID := range schedule.ScriptExecutionJob.Endpoints {

endpoint, err := handler.EndpointService.Endpoint(ID)
if err != nil {
return err
}

if endpoint.Type != portainer.EdgeAgentEnvironment {
nonEdgeEndpointIDs = append(nonEdgeEndpointIDs, endpoint.ID)
} else {
edgeEndpointIDs = append(edgeEndpointIDs, endpoint.ID)
}
}

if len(edgeEndpointIDs) > 0 {
edgeSchedule := &portainer.EdgeSchedule{
ID: schedule.ID,
CronExpression: strings.Join(edgeCronExpression, " "),
Script: base64.RawStdEncoding.EncodeToString(file),
Endpoints: edgeEndpointIDs,
Version: 1,
}

for _, endpointID := range edgeEndpointIDs {
handler.ReverseTunnelService.AddSchedule(endpointID, edgeSchedule)
}

schedule.EdgeSchedule = edgeSchedule
}

schedule.ScriptExecutionJob.Endpoints = nonEdgeEndpointIDs

scriptPath, err := handler.FileService.StoreScheduledJobFileFromBytes(strconv.Itoa(int(schedule.ID)), file)
if err != nil {
return err


+ 2
- 0
api/http/handler/schedules/schedule_delete.go View File

@@ -42,6 +42,8 @@ func (handler *Handler) scheduleDelete(w http.ResponseWriter, r *http.Request) *
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove the files associated to the schedule on the filesystem", err}
}

handler.ReverseTunnelService.RemoveSchedule(schedule.ID)

handler.JobScheduler.UnscheduleJob(schedule.ID)

err = handler.ScheduleService.DeleteSchedule(portainer.ScheduleID(scheduleID))


+ 19
- 0
api/http/handler/schedules/schedule_tasks.go View File

@@ -3,6 +3,7 @@ package schedules
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"

@@ -18,6 +19,7 @@ type taskContainer struct {
Status string `json:"Status"`
Created float64 `json:"Created"`
Labels map[string]string `json:"Labels"`
Edge bool `json:"Edge"`
}

// GET request on /api/schedules/:id/tasks
@@ -64,6 +66,22 @@ func (handler *Handler) scheduleTasks(w http.ResponseWriter, r *http.Request) *h
tasks = append(tasks, endpointTasks...)
}

if schedule.EdgeSchedule != nil {
for _, endpointID := range schedule.EdgeSchedule.Endpoints {

cronTask := taskContainer{
ID: fmt.Sprintf("schedule_%d", schedule.EdgeSchedule.ID),
EndpointID: endpointID,
Edge: true,
Status: "",
Created: 0,
Labels: map[string]string{},
}

tasks = append(tasks, cronTask)
}
}

return response.JSON(w, tasks)
}

@@ -87,6 +105,7 @@ func extractTasksFromContainerSnasphot(endpoint *portainer.Endpoint, scheduleID
for _, container := range containers {
if container.Labels["io.portainer.schedule.id"] == strconv.Itoa(int(scheduleID)) {
container.EndpointID = endpoint.ID
container.Edge = false
endpointTasks = append(endpointTasks, container)
}
}


+ 50
- 1
api/http/handler/schedules/schedule_update.go View File

@@ -1,6 +1,7 @@
package schedules

import (
"encoding/base64"
"errors"
"net/http"
"strconv"
@@ -58,7 +59,15 @@ func (handler *Handler) scheduleUpdate(w http.ResponseWriter, r *http.Request) *
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a schedule with the specified identifier inside the database", err}
}

updateJobSchedule := updateSchedule(schedule, &payload)
updateJobSchedule := false
if schedule.EdgeSchedule != nil {
err := handler.updateEdgeSchedule(schedule, &payload)
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update Edge schedule", err}
}
} else {
updateJobSchedule = updateSchedule(schedule, &payload)
}

if payload.FileContent != nil {
_, err := handler.FileService.StoreScheduledJobFileFromBytes(strconv.Itoa(scheduleID), []byte(*payload.FileContent))
@@ -85,6 +94,46 @@ func (handler *Handler) scheduleUpdate(w http.ResponseWriter, r *http.Request) *
return response.JSON(w, schedule)
}

func (handler *Handler) updateEdgeSchedule(schedule *portainer.Schedule, payload *scheduleUpdatePayload) error {
if payload.Name != nil {
schedule.Name = *payload.Name
}

if payload.Endpoints != nil {

edgeEndpointIDs := make([]portainer.EndpointID, 0)

for _, ID := range payload.Endpoints {
endpoint, err := handler.EndpointService.Endpoint(ID)
if err != nil {
return err
}

if endpoint.Type == portainer.EdgeAgentEnvironment {
edgeEndpointIDs = append(edgeEndpointIDs, endpoint.ID)
}
}

schedule.EdgeSchedule.Endpoints = edgeEndpointIDs
}

if payload.CronExpression != nil {
schedule.EdgeSchedule.CronExpression = *payload.CronExpression
schedule.EdgeSchedule.Version++
}

if payload.FileContent != nil {
schedule.EdgeSchedule.Script = base64.RawStdEncoding.EncodeToString([]byte(*payload.FileContent))
schedule.EdgeSchedule.Version++
}

for _, endpointID := range schedule.EdgeSchedule.Endpoints {
handler.ReverseTunnelService.AddSchedule(endpointID, schedule.EdgeSchedule)
}

return nil
}

func updateSchedule(schedule *portainer.Schedule, payload *scheduleUpdatePayload) bool {
updateJobSchedule := false



+ 5
- 0
api/http/handler/settings/settings_update.go View File

@@ -22,6 +22,7 @@ type settingsUpdatePayload struct {
EnableHostManagementFeatures *bool
SnapshotInterval *string
TemplatesURL *string
EdgeAgentCheckinInterval *int
}

func (payload *settingsUpdatePayload) Validate(r *http.Request) error {
@@ -103,6 +104,10 @@ func (handler *Handler) settingsUpdate(w http.ResponseWriter, r *http.Request) *
}
}

if payload.EdgeAgentCheckinInterval != nil {
settings.EdgeAgentCheckinInterval = *payload.EdgeAgentCheckinInterval
}

tlsError := handler.updateTLS(settings)
if tlsError != nil {
return tlsError


+ 4
- 2
api/http/handler/websocket/attach.go View File

@@ -62,8 +62,10 @@ func (handler *Handler) handleAttachRequest(w http.ResponseWriter, r *http.Reque

r.Header.Del("Origin")

if params.nodeName != "" || params.endpoint.Type == portainer.AgentOnDockerEnvironment {
return handler.proxyWebsocketRequest(w, r, params)
if params.endpoint.Type == portainer.AgentOnDockerEnvironment {
return handler.proxyAgentWebsocketRequest(w, r, params)
} else if params.endpoint.Type == portainer.EdgeAgentEnvironment {
return handler.proxyEdgeAgentWebsocketRequest(w, r, params)
}

websocketConn, err := handler.connectionUpgrader.Upgrade(w, r, nil)


+ 4
- 2
api/http/handler/websocket/exec.go View File

@@ -68,8 +68,10 @@ func (handler *Handler) websocketExec(w http.ResponseWriter, r *http.Request) *h
func (handler *Handler) handleExecRequest(w http.ResponseWriter, r *http.Request, params *webSocketRequestParams) error {
r.Header.Del("Origin")

if params.nodeName != "" || params.endpoint.Type == portainer.AgentOnDockerEnvironment {
return handler.proxyWebsocketRequest(w, r, params)
if params.endpoint.Type == portainer.AgentOnDockerEnvironment {
return handler.proxyAgentWebsocketRequest(w, r, params)
} else if params.endpoint.Type == portainer.EdgeAgentEnvironment {
return handler.proxyEdgeAgentWebsocketRequest(w, r, params)
}

websocketConn, err := handler.connectionUpgrader.Upgrade(w, r, nil)


+ 5
- 4
api/http/handler/websocket/handler.go View File

@@ -11,10 +11,11 @@ import (
// Handler is the HTTP handler used to handle websocket operations.
type Handler struct {
*mux.Router
EndpointService portainer.EndpointService
SignatureService portainer.DigitalSignatureService
requestBouncer *security.RequestBouncer
connectionUpgrader websocket.Upgrader
EndpointService portainer.EndpointService
SignatureService portainer.DigitalSignatureService
ReverseTunnelService portainer.ReverseTunnelService
requestBouncer *security.RequestBouncer
connectionUpgrader websocket.Upgrader
}

// NewHandler creates a handler to manage websocket operations.


+ 26
- 3
api/http/handler/websocket/proxy.go View File

@@ -2,14 +2,37 @@ package websocket

import (
"crypto/tls"
"fmt"
"net/http"
"net/url"

"github.com/gorilla/websocket"
"github.com/koding/websocketproxy"
"github.com/portainer/portainer/api"
"net/http"
"net/url"
)

func (handler *Handler) proxyWebsocketRequest(w http.ResponseWriter, r *http.Request, params *webSocketRequestParams) error {
func (handler *Handler) proxyEdgeAgentWebsocketRequest(w http.ResponseWriter, r *http.Request, params *webSocketRequestParams) error {
tunnel := handler.ReverseTunnelService.GetTunnelDetails(params.endpoint.ID)

endpointURL, err := url.Parse(fmt.Sprintf("http://localhost:%d", tunnel.Port))
if err != nil {
return err
}

endpointURL.Scheme = "ws"
proxy := websocketproxy.NewProxy(endpointURL)

proxy.Director = func(incoming *http.Request, out http.Header) {
out.Set(portainer.PortainerAgentTargetHeader, params.nodeName)
}

handler.ReverseTunnelService.SetTunnelStatusToActive(params.endpoint.ID)
proxy.ServeHTTP(w, r)

return nil
}

func (handler *Handler) proxyAgentWebsocketRequest(w http.ResponseWriter, r *http.Request, params *webSocketRequestParams) error {
agentURL, err := url.Parse(params.endpoint.URL)
if err != nil {
return err


+ 15
- 1
api/http/proxy/docker_transport.go View File

@@ -24,7 +24,9 @@ type (
DockerHubService portainer.DockerHubService
SettingsService portainer.SettingsService
SignatureService portainer.DigitalSignatureService
ReverseTunnelService portainer.ReverseTunnelService
endpointIdentifier portainer.EndpointID
endpointType portainer.EndpointType
}
restrictedDockerOperationContext struct {
isAdmin bool
@@ -58,7 +60,19 @@ func (p *proxyTransport) RoundTrip(request *http.Request) (*http.Response, error
}

func (p *proxyTransport) executeDockerRequest(request *http.Request) (*http.Response, error) {
return p.dockerTransport.RoundTrip(request)
response, err := p.dockerTransport.RoundTrip(request)

if p.endpointType != portainer.EdgeAgentEnvironment {
return response, err
}

if err == nil {
p.ReverseTunnelService.SetTunnelStatusToActive(p.endpointIdentifier)
} else {
p.ReverseTunnelService.SetTunnelStatusToIdle(p.endpointIdentifier)
}

return response, err
}

func (p *proxyTransport) proxyDockerRequest(request *http.Request) (*http.Response, error) {


+ 17
- 8
api/http/proxy/factory.go View File

@@ -21,6 +21,7 @@ type proxyFactory struct {
RegistryService portainer.RegistryService
DockerHubService portainer.DockerHubService
SignatureService portainer.DigitalSignatureService
ReverseTunnelService portainer.ReverseTunnelService
}

func (factory *proxyFactory) newHTTPProxy(u *url.URL) http.Handler {
@@ -29,21 +30,21 @@ func (factory *proxyFactory) newHTTPProxy(u *url.URL) http.Handler {
}

func newAzureProxy(credentials *portainer.AzureCredentials) (http.Handler, error) {
url, err := url.Parse(AzureAPIBaseURL)
remoteURL, err := url.Parse(AzureAPIBaseURL)
if err != nil {
return nil, err
}

proxy := newSingleHostReverseProxyWithHostHeader(url)
proxy := newSingleHostReverseProxyWithHostHeader(remoteURL)
proxy.Transport = NewAzureTransport(credentials)

return proxy, nil
}

func (factory *proxyFactory) newDockerHTTPSProxy(u *url.URL, tlsConfig *portainer.TLSConfiguration, enableSignature bool, endpointID portainer.EndpointID) (http.Handler, error) {
func (factory *proxyFactory) newDockerHTTPSProxy(u *url.URL, tlsConfig *portainer.TLSConfiguration, endpoint *portainer.Endpoint) (http.Handler, error) {
u.Scheme = "https"

proxy := factory.createDockerReverseProxy(u, enableSignature, endpointID)
proxy := factory.createDockerReverseProxy(u, endpoint)
config, err := crypto.CreateTLSConfigurationFromDisk(tlsConfig.TLSCACertPath, tlsConfig.TLSCertPath, tlsConfig.TLSKeyPath, tlsConfig.TLSSkipVerify)
if err != nil {
return nil, err
@@ -53,13 +54,19 @@ func (factory *proxyFactory) newDockerHTTPSProxy(u *url.URL, tlsConfig *portaine
return proxy, nil
}

func (factory *proxyFactory) newDockerHTTPProxy(u *url.URL, enableSignature bool, endpointID portainer.EndpointID) http.Handler {
func (factory *proxyFactory) newDockerHTTPProxy(u *url.URL, endpoint *portainer.Endpoint) http.Handler {
u.Scheme = "http"
return factory.createDockerReverseProxy(u, enableSignature, endpointID)
return factory.createDockerReverseProxy(u, endpoint)
}

func (factory *proxyFactory) createDockerReverseProxy(u *url.URL, enableSignature bool, endpointID portainer.EndpointID) *httputil.ReverseProxy {
func (factory *proxyFactory) createDockerReverseProxy(u *url.URL, endpoint *portainer.Endpoint) *httputil.ReverseProxy {
proxy := newSingleHostReverseProxyWithHostHeader(u)

enableSignature := false
if endpoint.Type == portainer.AgentOnDockerEnvironment {
enableSignature = true
}

transport := &proxyTransport{
enableSignature: enableSignature,
ResourceControlService: factory.ResourceControlService,
@@ -67,8 +74,10 @@ func (factory *proxyFactory) createDockerReverseProxy(u *url.URL, enableSignatur
SettingsService: factory.SettingsService,
RegistryService: factory.RegistryService,
DockerHubService: factory.DockerHubService,
ReverseTunnelService: factory.ReverseTunnelService,
dockerTransport: &http.Transport{},
endpointIdentifier: endpointID,
endpointIdentifier: endpoint.ID,
endpointType: endpoint.Type,
}

if enableSignature {


+ 4
- 2
api/http/proxy/factory_local.go View File

@@ -8,7 +8,7 @@ import (
portainer "github.com/portainer/portainer/api"
)

func (factory *proxyFactory) newLocalProxy(path string, endpointID portainer.EndpointID) http.Handler {
func (factory *proxyFactory) newLocalProxy(path string, endpoint *portainer.Endpoint) http.Handler {
proxy := &localProxy{}
transport := &proxyTransport{
enableSignature: false,
@@ -18,7 +18,9 @@ func (factory *proxyFactory) newLocalProxy(path string, endpointID portainer.End
RegistryService: factory.RegistryService,
DockerHubService: factory.DockerHubService,
dockerTransport: newSocketTransport(path),
endpointIdentifier: endpointID,
ReverseTunnelService: factory.ReverseTunnelService,
endpointIdentifier: endpoint.ID,
endpointType: endpoint.Type,
}
proxy.Transport = transport
return proxy


+ 5
- 4
api/http/proxy/factory_local_windows.go View File

@@ -5,12 +5,11 @@ package proxy
import (
"net"
"net/http"
"github.com/Microsoft/go-winio"

portainer "github.com/portainer/portainer/api"
)

func (factory *proxyFactory) newLocalProxy(path string, endpointID portainer.EndpointID) http.Handler {
func (factory *proxyFactory) newLocalProxy(path string, endpoint *portainer.Endpoint) http.Handler {
proxy := &localProxy{}
transport := &proxyTransport{
enableSignature: false,
@@ -19,8 +18,10 @@ func (factory *proxyFactory) newLocalProxy(path string, endpointID portainer.End
SettingsService: factory.SettingsService,
RegistryService: factory.RegistryService,
DockerHubService: factory.DockerHubService,
ReverseTunnelService: factory.ReverseTunnelService,
dockerTransport: newNamedPipeTransport(path),
endpointIdentifier: endpointID,
endpointIdentifier: endpoint.ID,
endpointType: endpoint.Type,
}
proxy.Transport = transport
return proxy


+ 35
- 18
api/http/proxy/manager.go View File

@@ -1,6 +1,7 @@
package proxy

import (
"fmt"
"net/http"
"net/url"
"strconv"
@@ -21,6 +22,7 @@ type (
// Manager represents a service used to manage Docker proxies.
Manager struct {
proxyFactory *proxyFactory
reverseTunnelService portainer.ReverseTunnelService
proxies cmap.ConcurrentMap
extensionProxies cmap.ConcurrentMap
legacyExtensionProxies cmap.ConcurrentMap
@@ -34,6 +36,7 @@ type (
RegistryService portainer.RegistryService
DockerHubService portainer.DockerHubService
SignatureService portainer.DigitalSignatureService
ReverseTunnelService portainer.ReverseTunnelService
}
)

@@ -50,13 +53,15 @@ func NewManager(parameters *ManagerParams) *Manager {
RegistryService: parameters.RegistryService,
DockerHubService: parameters.DockerHubService,
SignatureService: parameters.SignatureService,
ReverseTunnelService: parameters.ReverseTunnelService,
},
reverseTunnelService: parameters.ReverseTunnelService,
}
}

// GetProxy returns the proxy associated to a key
func (manager *Manager) GetProxy(key string) http.Handler {
proxy, ok := manager.proxies.Get(key)
func (manager *Manager) GetProxy(endpoint *portainer.Endpoint) http.Handler {
proxy, ok := manager.proxies.Get(string(endpoint.ID))
if !ok {
return nil
}
@@ -76,8 +81,8 @@ func (manager *Manager) CreateAndRegisterProxy(endpoint *portainer.Endpoint) (ht
}

// DeleteProxy deletes the proxy associated to a key
func (manager *Manager) DeleteProxy(key string) {
manager.proxies.Remove(key)
func (manager *Manager) DeleteProxy(endpoint *portainer.Endpoint) {
manager.proxies.Remove(string(endpoint.ID))
}

// GetExtensionProxy returns an extension proxy associated to an extension identifier
@@ -136,28 +141,40 @@ func (manager *Manager) CreateLegacyExtensionProxy(key, extensionAPIURL string)
return proxy, nil
}

func (manager *Manager) createDockerProxy(endpointURL *url.URL, tlsConfig *portainer.TLSConfiguration, endpointID portainer.EndpointID) (http.Handler, error) {
if endpointURL.Scheme == "tcp" {
if tlsConfig.TLS || tlsConfig.TLSSkipVerify {
return manager.proxyFactory.newDockerHTTPSProxy(endpointURL, tlsConfig, false, endpointID)
}
return manager.proxyFactory.newDockerHTTPProxy(endpointURL, false, endpointID), nil
func (manager *Manager) createDockerProxy(endpoint *portainer.Endpoint) (http.Handler, error) {
baseURL := endpoint.URL
if endpoint.Type == portainer.EdgeAgentEnvironment {
tunnel := manager.reverseTunnelService.GetTunnelDetails(endpoint.ID)
baseURL = fmt.Sprintf("http://localhost:%d", tunnel.Port)
}
return manager.proxyFactory.newLocalProxy(endpointURL.Path, endpointID), nil
}

func (manager *Manager) createProxy(endpoint *portainer.Endpoint) (http.Handler, error) {
endpointURL, err := url.Parse(endpoint.URL)
endpointURL, err := url.Parse(baseURL)
if err != nil {
return nil, err
}