diff --git a/src/main.rs b/src/main.rs --- a/src/main.rs +++ b/src/main.rs @@ -1,381 +1,386 @@ -use std::error::Error; -use std::net::SocketAddr; -use std::path::PathBuf; -use std::sync::Arc; - -use clap::{Parser, Subcommand, ValueEnum}; -use syncode_control::action_delivery::router as action_delivery_router; -use syncode_control::action_oci::PinnedOciResolver; -use syncode_control::action_repository::NativeActionRepository; -use syncode_control::action_store::FileActionStore; -use syncode_control::actions::ActionResolver; -use syncode_control::actions_read::ActionsRead; -use syncode_control::actions_read_identity::IdentityActionsAuthorization; -use syncode_control::admin::{Admin, router as admin_router}; -use syncode_control::check_events::{CheckEventPublisher, CheckEventPublisherConfig}; -use syncode_control::checks::Checks; -use syncode_control::maintenance; -use syncode_control::projection; -use syncode_control::repository::{RepositoryContents, RepositorySecrets}; -use syncode_control::repository_grpc::NativeRepositoryContents; -use syncode_control::repository_sources::RepositorySources; -use syncode_control::secrets::RuntimeSecrets; -use syncode_control::token::EnrolmentScope; -use syncode_control::webhook::{Intake, router}; -use syncode_control_node::{ - ArtifactTokenAuthority, CapabilityAuthority, GeneratedActionsReadServer, GeneratedChecksServer, - GeneratedSecretsServer, GeneratedServer, NodeSessionServer, ProjectionClient, -}; -use syncode_control_nodes::{Nodes, Scope}; -use syncode_control_runs::{ - AuditConfiguration, AuditControlMode, AuditEvent, AuditPayload, NodeId, Runs, SchedulerPolicy, -}; -use syncode_control_store::Postgres; -use tokio::net::TcpListener; -use tonic::transport::Server; -use url::Url; - -#[derive(Debug, Parser)] -#[command(name = "syncode-control", version, about)] -struct Arguments { - #[command(subcommand)] - command: Option, - - /// Where nodes open their session. - #[arg(long, default_value = "127.0.0.1:8090")] - listen: SocketAddr, - - /// Where the native repository event feed delivers events. - #[arg(long, default_value = "127.0.0.1:8091")] - listen_events: SocketAddr, - - /// The repository service this control plane reads workflows from. - #[arg(long, env = "SYNCODE_REPOSITORY_SOURCE_URL")] - repository_source: Url, - - /// A token allowed to read repository contents. - #[arg(long, env = "SYNCODE_REPOSITORY_SOURCE_TOKEN", hide_env_values = true)] - repository_source_token: String, - - /// Where native repositories are read over gRPC. - #[arg(long, env = "SYNCODE_REPOSITORY_GRPC_URL")] - repository_grpc: String, - - #[arg(long, env = "SYNCODE_IDENTITY_GRPC_URL")] - identity_grpc: String, - - #[arg(long, env = "SYNCODE_IDENTITY_SHARED_SECRET", hide_env_values = true)] - identity_shared_secret: String, - - #[arg(long, env = "SYNCODE_ACTION_MIRROR_URL")] - action_mirror_url: Url, - - #[arg( - long, - env = "SYNCODE_ACTION_ALLOWLIST", - value_delimiter = ',', - num_args = 1.. - )] - action_allowlist: Vec, - - #[arg( - long, - env = "SYNCODE_ACTION_OCI_REGISTRIES", - value_delimiter = ',', - num_args = 1.. - )] - action_oci_registries: Vec, - - #[arg(long, env = "SYNCODE_ACTION_STORE")] - action_store: PathBuf, - - #[arg(long, env = "SYNCODE_ACTION_ARTIFACT_PUBLIC_URL")] - action_artifact_public_url: Url, - - #[arg(long, env = "SYNCODE_SECRET_SOURCE_URL")] - secret_source: Url, - - #[arg(long, env = "SYNCODE_SECRET_SOURCE_TOKEN", hide_env_values = true)] - secret_source_token: String, - - #[arg(long, env = "SYNCODE_CAPABILITY_SIGNING_KEY", hide_env_values = true)] - capability_signing_key: String, - - #[arg(long, env = "SYNCODE_ARTIFACT_SIGNING_KEY", hide_env_values = true)] - artifact_signing_key: String, - - #[arg(long, env = "SYNCODE_PROJECTION_URL")] - projection_url: Url, - - #[arg(long, env = "SYNCODE_PROJECTION_TOKEN", hide_env_values = true)] - projection_token: String, - - #[arg(long, env = "SYNCODE_PROJECTION_MIN_RUN_NUMBER")] - projection_min_run_number: u64, - - /// The secret used to sign the native event feed. - #[arg(long, env = "SYNCODE_EVENT_FEED_SECRET", hide_env_values = true)] - event_feed_secret: String, - - #[arg(long, env = "SYNCODE_COLLAB_EVENT_URL")] - collaboration_event_url: Url, - - #[arg(long, env = "SYNCODE_COLLAB_EVENT_SECRET", hide_env_values = true)] - collaboration_event_secret: String, - - #[arg(long, env = "SYNCODE_CONTROL_SOURCE_NODE_ID")] - source_node_id: uuid::Uuid, - - #[arg( - long, - env = "SYNCODE_CHECK_EVENT_INTERVAL_SECONDS", - default_value_t = 1 - )] - check_event_interval_seconds: u64, - - #[arg(long, env = "SYNCODE_CHECK_EVENT_BATCH", default_value_t = 100)] - check_event_batch: i64, - - /// Bearer token protecting the operational API. - #[arg(long, env = "SYNCODE_CONTROL_ADMIN_TOKEN", hide_env_values = true)] - admin_token: Option, - - /// Where the run log is kept. - #[arg(long, env = "SYNCODE_DATABASE_URL", hide_env_values = true)] - database: String, - - #[arg(long, default_value_t = 8)] - database_connections: u32, - - #[arg(long, env = "SYNCODE_ORGANIZATION_CONCURRENCY", default_value_t = 100)] - organization_concurrency: u32, - - #[arg(long, env = "SYNCODE_REPOSITORY_CONCURRENCY", default_value_t = 20)] - repository_concurrency: u32, - - #[arg(long, env = "SYNCODE_PRINCIPAL_CONCURRENCY", default_value_t = 20)] - principal_concurrency: u32, - - #[arg( - long, - env = "SYNCODE_ORGANIZATION_QUEUE_QUOTA", - default_value_t = 10_000 - )] - organization_queue_quota: u32, - - #[arg(long, env = "SYNCODE_PRINCIPAL_QUEUE_QUOTA", default_value_t = 1_000)] - principal_queue_quota: u32, - - #[arg(long, env = "SYNCODE_AUDIT_RETENTION_DAYS", default_value_t = 90)] - audit_retention_days: u32, - - /// Whether compiled shadow runs may be assigned to nodes. - #[arg(long, value_enum, default_value_t = Mode::Shadow)] - mode: Mode, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -enum Mode { - Shadow, - Active, -} - -#[derive(Debug, Subcommand)] -enum Command { - /// Issue an enrolment token a node can spend for an identity. - /// - /// The token is stored before it is printed, so one that reaches an - /// operator is one the control plane will honour. - IssueToken { - /// How far the token reaches: `instance`, `organisation:` or - /// `repository:/`. - #[arg(long, default_value = "instance")] - scope: String, - }, - /// Revoke a node identity immediately. - RevokeNode { - /// The node UUID printed at enrolment. - node: String, - }, -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - let arguments = Arguments::parse(); - // The log is read before anything is served: a node that reconnects has to - // meet the run it was holding, not a control plane that has forgotten it. - let store = Postgres::connect(&arguments.database, arguments.database_connections).await?; - let policy = SchedulerPolicy::new( - arguments.organization_concurrency, - arguments.repository_concurrency, - arguments.principal_concurrency, - arguments.organization_queue_quota, - arguments.principal_queue_quota, - ); - let runs = Runs::restored_with_policy(store.clone(), policy).await?; - let nodes = Nodes::restored(store.clone()).await?; - - if let Some(command) = arguments.command { - match command { - Command::IssueToken { scope } => { - let scope: Scope = scope.parse::()?.into(); - let token = nodes.issue_token(scope).await?; - println!("{}", token.secret().expose()); - } - Command::RevokeNode { node } => { - nodes.revoke(node.parse::()?).await?; - } - } - return Ok(()); - } - - store - .record_audit(AuditEvent::new("control.configuration", "applied").after( - AuditPayload::Configuration(AuditConfiguration { - mode: match arguments.mode { - Mode::Shadow => AuditControlMode::Shadow, - Mode::Active => AuditControlMode::Active, - }, - organization_concurrency: arguments.organization_concurrency, - repository_concurrency: arguments.repository_concurrency, - principal_concurrency: arguments.principal_concurrency, - organization_queue_quota: arguments.organization_queue_quota, - principal_queue_quota: arguments.principal_queue_quota, - audit_retention_days: arguments.audit_retention_days, - projection_min_run_number: arguments.projection_min_run_number, - }), - )) - .await?; - tokio::spawn(maintenance::sweep(runs.clone(), nodes.clone())); - tokio::spawn(maintenance::retain_audit( - store.clone(), - arguments.audit_retention_days, - )); - - let action_mirror = arguments - .action_mirror_url - .to_string() - .parse::()?; - let action_allowlist = arguments - .action_allowlist - .into_iter() - .map(|repository| { - repository - .to_string() - .parse::() - }) - .collect::, _>>()?; - let action_store = FileActionStore::new(arguments.action_store); - let action_repository = NativeActionRepository::connect( - arguments.repository_grpc.clone(), - arguments.identity_grpc.clone(), - arguments.identity_shared_secret.clone(), - ) - .await?; - let action_resolver = ActionResolver::new( - action_repository, - action_store.clone(), - PinnedOciResolver::new(arguments.action_oci_registries)?, - action_mirror, - action_allowlist, - ); - let native_repository = - NativeRepositoryContents::connect(arguments.repository_grpc.clone()).await?; - let repository_sources = RepositorySources::new( - native_repository, - RepositoryContents::new( - arguments.repository_source, - arguments.repository_source_token, - ), - ); - let intake = Arc::new(Intake::new( - repository_sources, - runs.clone(), - action_resolver, - arguments.event_feed_secret, - )); - let admin_token = arguments - .admin_token - .filter(|token| !token.is_empty()) - .ok_or("SYNCODE_CONTROL_ADMIN_TOKEN is required while serving")?; - let artifact_authority = ArtifactTokenAuthority::new(arguments.artifact_signing_key)?; - let http = router(intake) - .merge(admin_router(Arc::new( - Admin::new(runs.clone(), nodes.clone(), admin_token).with_operations(store.clone()), - ))) - .merge(action_delivery_router( - action_store, - artifact_authority.clone(), - runs.clone(), - )); - let events = TcpListener::bind(arguments.listen_events).await?; - let check_events = CheckEventPublisher::new( - store.clone(), - CheckEventPublisherConfig { - endpoint: arguments.collaboration_event_url.as_str(), - secret: arguments.collaboration_event_secret, - source_node_id: arguments.source_node_id, - interval: std::time::Duration::from_secs(arguments.check_event_interval_seconds), - batch: arguments.check_event_batch, - }, - )?; - - let authority = CapabilityAuthority::new(arguments.capability_signing_key)?; - let projection = ProjectionClient::new(arguments.projection_url, arguments.projection_token)?; - let secret_service = RuntimeSecrets::new( - runs.clone(), - nodes.clone(), - RepositorySecrets::new(arguments.secret_source, arguments.secret_source_token), - authority.clone(), - ); - let actions_read = ActionsRead::new( - runs.clone(), - IdentityActionsAuthorization::connect( - arguments.identity_grpc, - arguments.identity_shared_secret, - ) - .await?, - ); - - // Both ends of the control plane run for as long as the other does: without - // events there is nothing to assign, and without sessions there is nobody to - // assign it to. Whichever stops first takes the process down with it. - let node_service = match arguments.mode { - Mode::Shadow => NodeSessionServer::shadow( - runs.clone(), - nodes, - authority, - artifact_authority, - arguments.action_artifact_public_url, - projection.clone(), - ), - Mode::Active => NodeSessionServer::new( - runs.clone(), - nodes, - authority, - artifact_authority, - arguments.action_artifact_public_url, - projection.clone(), - ), - }; - tokio::select! { - served = axum::serve(events, http).into_future() => served?, - served = projection::serve(runs.clone(), projection, arguments.projection_min_run_number) => served?, - published = check_events.run() => published?, - served = Server::builder() - .add_service(GeneratedServer::new(node_service)) - .add_service(GeneratedSecretsServer::new(secret_service)) - .add_service(GeneratedChecksServer::new(Checks::new(runs.clone()))) - .add_service(GeneratedActionsReadServer::new(actions_read)) - .serve_with_shutdown(arguments.listen, shutdown()) => served?, - } - - Ok(()) -} - -async fn shutdown() { - if let Err(error) = tokio::signal::ctrl_c().await { - eprintln!("cannot listen for shutdown, keeping the service running: {error}"); - std::future::pending::<()>().await; - } -} +use std::error::Error; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; + +use clap::{Parser, Subcommand, ValueEnum}; +use syncode_control::action_delivery::router as action_delivery_router; +use syncode_control::action_oci::PinnedOciResolver; +use syncode_control::action_repository::NativeActionRepository; +use syncode_control::action_store::FileActionStore; +use syncode_control::actions::ActionResolver; +use syncode_control::actions_read::ActionsRead; +use syncode_control::actions_read_identity::IdentityActionsAuthorization; +use syncode_control::admin::{Admin, router as admin_router}; +use syncode_control::check_events::{CheckEventPublisher, CheckEventPublisherConfig}; +use syncode_control::checks::Checks; +use syncode_control::maintenance; +use syncode_control::projection; +use syncode_control::repository::{RepositoryContents, RepositorySecrets}; +use syncode_control::repository_grpc::NativeRepositoryContents; +use syncode_control::repository_sources::RepositorySources; +use syncode_control::secrets::RuntimeSecrets; +use syncode_control::token::EnrolmentScope; +use syncode_control::webhook::{Intake, router}; +use syncode_control_node::{ + ArtifactTokenAuthority, CapabilityAuthority, GeneratedActionsReadServer, GeneratedChecksServer, + GeneratedSecretsServer, GeneratedServer, NodeSessionServer, ProjectionClient, +}; +use syncode_control_nodes::{Nodes, Scope}; +use syncode_control_runs::{ + AuditConfiguration, AuditControlMode, AuditEvent, AuditPayload, NodeId, Runs, SchedulerPolicy, +}; +use syncode_control_store::Postgres; +use tokio::net::TcpListener; +use tonic::transport::Server; +use url::Url; + +#[derive(Debug, Parser)] +#[command(name = "syncode-control", version, about)] +struct Arguments { + #[command(subcommand)] + command: Option, + + /// Where nodes open their session. + #[arg(long, default_value = "127.0.0.1:8090")] + listen: SocketAddr, + + /// Where the native repository event feed delivers events. + #[arg(long, default_value = "127.0.0.1:8091")] + listen_events: SocketAddr, + + /// The repository service this control plane reads workflows from. + #[arg(long, env = "SYNCODE_REPOSITORY_SOURCE_URL")] + repository_source: Url, + + /// A token allowed to read repository contents. + #[arg(long, env = "SYNCODE_REPOSITORY_SOURCE_TOKEN", hide_env_values = true)] + repository_source_token: String, + + /// Where native repositories are read over gRPC. + #[arg(long, env = "SYNCODE_REPOSITORY_GRPC_URL")] + repository_grpc: String, + + #[arg(long, env = "SYNCODE_IDENTITY_GRPC_URL")] + identity_grpc: String, + + #[arg(long, env = "SYNCODE_IDENTITY_SHARED_SECRET", hide_env_values = true)] + identity_shared_secret: String, + + #[arg(long, env = "SYNCODE_ACTION_MIRROR_URL")] + action_mirror_url: Url, + + #[arg( + long, + env = "SYNCODE_ACTION_ALLOWLIST", + value_delimiter = ',', + num_args = 1.. + )] + action_allowlist: Vec, + + #[arg( + long, + env = "SYNCODE_ACTION_OCI_REGISTRIES", + value_delimiter = ',', + num_args = 1.. + )] + action_oci_registries: Vec, + + #[arg(long, env = "SYNCODE_ACTION_STORE")] + action_store: PathBuf, + + #[arg(long, env = "SYNCODE_ACTION_ARTIFACT_PUBLIC_URL")] + action_artifact_public_url: Url, + + #[arg(long, env = "SYNCODE_SECRET_SOURCE_URL")] + secret_source: Url, + + #[arg(long, env = "SYNCODE_SECRET_SOURCE_TOKEN", hide_env_values = true)] + secret_source_token: String, + + #[arg(long, env = "SYNCODE_CAPABILITY_SIGNING_KEY", hide_env_values = true)] + capability_signing_key: String, + + #[arg(long, env = "SYNCODE_ARTIFACT_SIGNING_KEY", hide_env_values = true)] + artifact_signing_key: String, + + #[arg(long, env = "SYNCODE_PROJECTION_URL")] + projection_url: Url, + + #[arg(long, env = "SYNCODE_PROJECTION_TOKEN", hide_env_values = true)] + projection_token: String, + + #[arg(long, env = "SYNCODE_PROJECTION_MIN_RUN_NUMBER")] + projection_min_run_number: u64, + + /// The secret used to sign the native event feed. + #[arg(long, env = "SYNCODE_EVENT_FEED_SECRET", hide_env_values = true)] + event_feed_secret: String, + + #[arg(long, env = "SYNCODE_COLLAB_EVENT_URL")] + collaboration_event_url: Url, + + #[arg(long, env = "SYNCODE_COLLAB_EVENT_SECRET", hide_env_values = true)] + collaboration_event_secret: String, + + #[arg(long, env = "SYNCODE_CONTROL_SOURCE_NODE_ID")] + source_node_id: uuid::Uuid, + + #[arg( + long, + env = "SYNCODE_CHECK_EVENT_INTERVAL_SECONDS", + default_value_t = 1 + )] + check_event_interval_seconds: u64, + + #[arg(long, env = "SYNCODE_CHECK_EVENT_BATCH", default_value_t = 100)] + check_event_batch: i64, + + /// Bearer token protecting the operational API. + #[arg(long, env = "SYNCODE_CONTROL_ADMIN_TOKEN", hide_env_values = true)] + admin_token: Option, + + /// Where the run log is kept. + #[arg(long, env = "SYNCODE_DATABASE_URL", hide_env_values = true)] + database: String, + + #[arg(long, default_value_t = 8)] + database_connections: u32, + + #[arg(long, env = "SYNCODE_ORGANIZATION_CONCURRENCY", default_value_t = 100)] + organization_concurrency: u32, + + #[arg(long, env = "SYNCODE_REPOSITORY_CONCURRENCY", default_value_t = 20)] + repository_concurrency: u32, + + #[arg(long, env = "SYNCODE_PRINCIPAL_CONCURRENCY", default_value_t = 20)] + principal_concurrency: u32, + + #[arg( + long, + env = "SYNCODE_ORGANIZATION_QUEUE_QUOTA", + default_value_t = 10_000 + )] + organization_queue_quota: u32, + + #[arg(long, env = "SYNCODE_PRINCIPAL_QUEUE_QUOTA", default_value_t = 1_000)] + principal_queue_quota: u32, + + #[arg(long, env = "SYNCODE_AUDIT_RETENTION_DAYS", default_value_t = 90)] + audit_retention_days: u32, + + /// Whether compiled shadow runs may be assigned to nodes. + #[arg(long, value_enum, default_value_t = Mode::Shadow)] + mode: Mode, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +enum Mode { + Shadow, + Active, +} + +#[derive(Debug, Subcommand)] +enum Command { + /// Issue an enrolment token a node can spend for an identity. + /// + /// The token is stored before it is printed, so one that reaches an + /// operator is one the control plane will honour. + IssueToken { + /// How far the token reaches: `instance`, `organisation:` or + /// `repository:/`. + #[arg(long, default_value = "instance")] + scope: String, + }, + /// Revoke a node identity immediately. + RevokeNode { + /// The node UUID printed at enrolment. + node: String, + }, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let arguments = Arguments::parse(); + // The log is read before anything is served: a node that reconnects has to + // meet the run it was holding, not a control plane that has forgotten it. + let store = Postgres::connect(&arguments.database, arguments.database_connections).await?; + let policy = SchedulerPolicy::new( + arguments.organization_concurrency, + arguments.repository_concurrency, + arguments.principal_concurrency, + arguments.organization_queue_quota, + arguments.principal_queue_quota, + ); + let runs = Runs::restored_with_policy(store.clone(), policy).await?; + let nodes = Nodes::restored(store.clone()).await?; + + if let Some(command) = arguments.command { + match command { + Command::IssueToken { scope } => { + let scope: Scope = scope.parse::()?.into(); + let token = nodes.issue_token(scope).await?; + println!("{}", token.secret().expose()); + } + Command::RevokeNode { node } => { + nodes.revoke(node.parse::()?).await?; + } + } + return Ok(()); + } + + store + .record_audit(AuditEvent::new("control.configuration", "applied").after( + AuditPayload::Configuration(AuditConfiguration { + mode: match arguments.mode { + Mode::Shadow => AuditControlMode::Shadow, + Mode::Active => AuditControlMode::Active, + }, + organization_concurrency: arguments.organization_concurrency, + repository_concurrency: arguments.repository_concurrency, + principal_concurrency: arguments.principal_concurrency, + organization_queue_quota: arguments.organization_queue_quota, + principal_queue_quota: arguments.principal_queue_quota, + audit_retention_days: arguments.audit_retention_days, + projection_min_run_number: arguments.projection_min_run_number, + }), + )) + .await?; + tokio::spawn(maintenance::sweep(runs.clone(), nodes.clone())); + tokio::spawn(maintenance::retain_audit( + store.clone(), + arguments.audit_retention_days, + )); + + let action_mirror = arguments + .action_mirror_url + .to_string() + .parse::()?; + let action_allowlist = arguments + .action_allowlist + .into_iter() + .map(|repository| { + repository + .to_string() + .parse::() + }) + .collect::, _>>()?; + let action_store = FileActionStore::new(arguments.action_store); + let action_repository = NativeActionRepository::connect( + arguments.repository_grpc.clone(), + arguments.identity_grpc.clone(), + arguments.identity_shared_secret.clone(), + ) + .await?; + let action_resolver = ActionResolver::new( + action_repository, + action_store.clone(), + PinnedOciResolver::new(arguments.action_oci_registries)?, + action_mirror, + action_allowlist, + ); + let native_repository = + NativeRepositoryContents::connect(arguments.repository_grpc.clone()).await?; + let repository_sources = RepositorySources::new( + native_repository, + RepositoryContents::new( + arguments.repository_source, + arguments.repository_source_token, + ), + ); + let intake = Arc::new(Intake::new( + repository_sources, + runs.clone(), + action_resolver, + arguments.event_feed_secret, + )); + let admin_token = arguments + .admin_token + .filter(|token| !token.is_empty()) + .ok_or("SYNCODE_CONTROL_ADMIN_TOKEN is required while serving")?; + let artifact_authority = ArtifactTokenAuthority::new(arguments.artifact_signing_key)?; + let http = router(intake) + .merge(admin_router(Arc::new( + Admin::new(runs.clone(), nodes.clone(), admin_token).with_operations(store.clone()), + ))) + .merge(action_delivery_router( + action_store, + artifact_authority.clone(), + runs.clone(), + )); + let events = TcpListener::bind(arguments.listen_events).await?; + let check_events = CheckEventPublisher::new( + store.clone(), + CheckEventPublisherConfig { + endpoint: arguments.collaboration_event_url.as_str(), + secret: arguments.collaboration_event_secret, + source_node_id: arguments.source_node_id, + interval: std::time::Duration::from_secs(arguments.check_event_interval_seconds), + batch: arguments.check_event_batch, + }, + )?; + + let authority = CapabilityAuthority::new(arguments.capability_signing_key)?; + let projection = ProjectionClient::new( + arguments.projection_url, + arguments.projection_token, + arguments.identity_grpc.clone(), + arguments.identity_shared_secret.clone(), + )?; + let secret_service = RuntimeSecrets::new( + runs.clone(), + nodes.clone(), + RepositorySecrets::new(arguments.secret_source, arguments.secret_source_token), + authority.clone(), + ); + let actions_read = ActionsRead::new( + runs.clone(), + IdentityActionsAuthorization::connect( + arguments.identity_grpc, + arguments.identity_shared_secret, + ) + .await?, + ); + + // Both ends of the control plane run for as long as the other does: without + // events there is nothing to assign, and without sessions there is nobody to + // assign it to. Whichever stops first takes the process down with it. + let node_service = match arguments.mode { + Mode::Shadow => NodeSessionServer::shadow( + runs.clone(), + nodes, + authority, + artifact_authority, + arguments.action_artifact_public_url, + projection.clone(), + ), + Mode::Active => NodeSessionServer::new( + runs.clone(), + nodes, + authority, + artifact_authority, + arguments.action_artifact_public_url, + projection.clone(), + ), + }; + tokio::select! { + served = axum::serve(events, http).into_future() => served?, + served = projection::serve(runs.clone(), projection, arguments.projection_min_run_number) => served?, + published = check_events.run() => published?, + served = Server::builder() + .add_service(GeneratedServer::new(node_service)) + .add_service(GeneratedSecretsServer::new(secret_service)) + .add_service(GeneratedChecksServer::new(Checks::new(runs.clone()))) + .add_service(GeneratedActionsReadServer::new(actions_read)) + .serve_with_shutdown(arguments.listen, shutdown()) => served?, + } + + Ok(()) +} + +async fn shutdown() { + if let Err(error) = tokio::signal::ctrl_c().await { + eprintln!("cannot listen for shutdown, keeping the service running: {error}"); + std::future::pending::<()>().await; + } +} diff --git a/tests/action_repository.rs b/tests/action_repository.rs --- a/tests/action_repository.rs +++ b/tests/action_repository.rs @@ -1,121 +1,129 @@ -#![allow(clippy::expect_used)] - -#[path = "support/repository.rs"] -#[allow(dead_code)] -mod repository; - -use std::error::Error; - -use syncode_control::action_repository::NativeActionRepository; -use syncode_control::actions::ActionRepositoryPort; -use syncode_control_node::identity_wire::identity_server::{Identity, IdentityServer}; -use syncode_control_node::identity_wire::{ - CheckCapabilityRequest, CheckCapabilityResponse, ResolveRepositoryRequest, - ResolveRepositoryResponse, ValidateSessionRequest, ValidateSessionResponse, -}; -use tokio::net::TcpListener; -use tokio_stream::wrappers::TcpListenerStream; -use tonic::transport::Server; -use tonic::{Request, Response, Status}; - -type TestResult = Result>; - -#[derive(Default)] -struct FixtureIdentity; - -#[tonic::async_trait] -impl Identity for FixtureIdentity { - async fn validate_session( - &self, - _request: Request, - ) -> Result, Status> { - Err(Status::unimplemented("validate_session")) - } - - async fn check_capability( - &self, - _request: Request, - ) -> Result, Status> { - Err(Status::unimplemented("check_capability")) - } - - async fn resolve_repository( - &self, - request: Request, - ) -> Result, Status> { - if request - .metadata() - .get("authorization") - .and_then(|value| value.to_str().ok()) - != Some("Bearer shared-secret") - { - return Err(Status::unauthenticated("missing authorization")); - } - let request = request.into_inner(); - if request.owner != "actions" || request.name != "checkout" { - return Err(Status::not_found("repository")); - } - Ok(Response::new(ResolveRepositoryResponse { - repository_id: repository::REPOSITORY.to_owned(), - })) - } -} - -async fn source() -> TestResult { - let repository_listener = TcpListener::bind("127.0.0.1:0").await?; - let repository_endpoint = format!("http://{}", repository_listener.local_addr()?); - tokio::spawn(async move { - let _ = Server::builder() - .add_service(repository::service()) - .serve_with_incoming(TcpListenerStream::new(repository_listener)) - .await; - }); - - let identity_listener = TcpListener::bind("127.0.0.1:0").await?; - let identity_endpoint = format!("http://{}", identity_listener.local_addr()?); - tokio::spawn(async move { - let _ = Server::builder() - .add_service(IdentityServer::new(FixtureIdentity)) - .serve_with_incoming(TcpListenerStream::new(identity_listener)) - .await; - }); - - Ok(NativeActionRepository::connect( - repository_endpoint, - identity_endpoint, - "shared-secret".to_owned(), - ) - .await?) -} - -#[tokio::test] -async fn resolves_coordinates_and_fetches_an_immutable_native_archive() -> TestResult { - let source = source().await?; - let snapshot = source - .fetch( - "https://dev.syncode.sh/actions/checkout".parse()?, - "v4".to_owned(), - ) - .await?; - - assert_eq!(snapshot.commit.as_ref(), repository::COMMIT); - let paths = tar::Archive::new(snapshot.archive.as_slice()) - .entries()? - .map(|entry| entry.and_then(|entry| entry.path().map(|path| path.into_owned()))) - .collect::, _>>()?; - assert_eq!(paths, [std::path::PathBuf::from("source/action.yml")]); - Ok(()) -} - -#[tokio::test] -async fn accepts_a_native_repository_id_without_coordinate_resolution() -> TestResult { - let source = source().await?; - let snapshot = source - .fetch( - format!("https://dev.syncode.sh/{}", repository::REPOSITORY).parse()?, - repository::COMMIT.to_owned(), - ) - .await?; - assert_eq!(snapshot.commit.as_ref(), repository::COMMIT); - Ok(()) -} +#![allow(clippy::expect_used)] + +#[path = "support/repository.rs"] +#[allow(dead_code)] +mod repository; + +use std::error::Error; + +use syncode_control::action_repository::NativeActionRepository; +use syncode_control::actions::ActionRepositoryPort; +use syncode_control_node::identity_wire::identity_server::{Identity, IdentityServer}; +use syncode_control_node::identity_wire::{ + CheckCapabilityRequest, CheckCapabilityResponse, GetRepositoryCoordinatesRequest, + GetRepositoryCoordinatesResponse, ResolveRepositoryRequest, ResolveRepositoryResponse, + ValidateSessionRequest, ValidateSessionResponse, +}; +use tokio::net::TcpListener; +use tokio_stream::wrappers::TcpListenerStream; +use tonic::transport::Server; +use tonic::{Request, Response, Status}; + +type TestResult = Result>; + +#[derive(Default)] +struct FixtureIdentity; + +#[tonic::async_trait] +impl Identity for FixtureIdentity { + async fn get_repository_coordinates( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("get_repository_coordinates")) + } + + async fn validate_session( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("validate_session")) + } + + async fn check_capability( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("check_capability")) + } + + async fn resolve_repository( + &self, + request: Request, + ) -> Result, Status> { + if request + .metadata() + .get("authorization") + .and_then(|value| value.to_str().ok()) + != Some("Bearer shared-secret") + { + return Err(Status::unauthenticated("missing authorization")); + } + let request = request.into_inner(); + if request.owner != "actions" || request.name != "checkout" { + return Err(Status::not_found("repository")); + } + Ok(Response::new(ResolveRepositoryResponse { + repository_id: repository::REPOSITORY.to_owned(), + })) + } +} + +async fn source() -> TestResult { + let repository_listener = TcpListener::bind("127.0.0.1:0").await?; + let repository_endpoint = format!("http://{}", repository_listener.local_addr()?); + tokio::spawn(async move { + let _ = Server::builder() + .add_service(repository::service()) + .serve_with_incoming(TcpListenerStream::new(repository_listener)) + .await; + }); + + let identity_listener = TcpListener::bind("127.0.0.1:0").await?; + let identity_endpoint = format!("http://{}", identity_listener.local_addr()?); + tokio::spawn(async move { + let _ = Server::builder() + .add_service(IdentityServer::new(FixtureIdentity)) + .serve_with_incoming(TcpListenerStream::new(identity_listener)) + .await; + }); + + Ok(NativeActionRepository::connect( + repository_endpoint, + identity_endpoint, + "shared-secret".to_owned(), + ) + .await?) +} + +#[tokio::test] +async fn resolves_coordinates_and_fetches_an_immutable_native_archive() -> TestResult { + let source = source().await?; + let snapshot = source + .fetch( + "https://dev.syncode.sh/actions/checkout".parse()?, + "v4".to_owned(), + ) + .await?; + + assert_eq!(snapshot.commit.as_ref(), repository::COMMIT); + let paths = tar::Archive::new(snapshot.archive.as_slice()) + .entries()? + .map(|entry| entry.and_then(|entry| entry.path().map(|path| path.into_owned()))) + .collect::, _>>()?; + assert_eq!(paths, [std::path::PathBuf::from("source/action.yml")]); + Ok(()) +} + +#[tokio::test] +async fn accepts_a_native_repository_id_without_coordinate_resolution() -> TestResult { + let source = source().await?; + let snapshot = source + .fetch( + format!("https://dev.syncode.sh/{}", repository::REPOSITORY).parse()?, + repository::COMMIT.to_owned(), + ) + .await?; + assert_eq!(snapshot.commit.as_ref(), repository::COMMIT); + Ok(()) +} diff --git a/tests/node_session.rs b/tests/node_session.rs --- a/tests/node_session.rs +++ b/tests/node_session.rs @@ -1,1638 +1,1644 @@ -#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] - -#[path = "support/actions.rs"] -mod actions; - -use std::error::Error; -use std::time::{Duration, SystemTime}; - -use syncode_control::trigger::{Triggered, trigger}; -use syncode_control_node::wire::{ - Acknowledgement, Capabilities, Capacity, ControlMessage, Enrol, Heartbeat, Hello, JobProgress, - NodeMessage, control_message, node_message, -}; -use syncode_control_node::{ - ArtifactTokenAuthority, CapabilityAuthority, GeneratedServer, NodeSessionClient, - NodeSessionServer, ProjectionClient, -}; -use syncode_control_nodes::{CREDENTIAL_TERM, Ephemeral, Lifecycle, Nodes, Scope}; -use syncode_control_runs::{ - Conclusion, Fence, Forgotten, JobId, MatrixPolicy, Origin, Priority, QueuedJob, Requirements, - RunState, Runs, -}; -use syncode_workflow::{ - Event, EventKind, ExecutionPlan, PlanSchemaVersion, VersionedPlan, WorkflowCompiler, - WorkflowDialect, WorkflowSource, -}; -use syncode_workflow_github_actions::compiler::GithubActionsCompiler; -use syncode_workflow_github_actions::expression::ExpressionProgram; -use tokio::net::TcpListener; -use tokio::sync::mpsc; -use tokio_stream::StreamExt; -use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream}; -use tonic::Streaming; -use tonic::transport::Server; - -use actions::FIXTURE_ACTIONS; - -type TestResult = Result>; - -const WORKFLOW: &str = r#" -name: CI -jobs: - build: - runs-on: [self-hosted, linux] - steps: - - run: echo "the plan came from the control plane" -"#; - -fn compile() -> ExecutionPlan { - let hir = GithubActionsCompiler - .compile(&WorkflowSource::new( - WorkflowDialect::GitHubActions, - WORKFLOW.as_bytes().to_vec(), - )) - .unwrap_or_else(|error| panic!("compile: {error}")); - syncode_workflow::plans(hir) - .unwrap_or_else(|error| panic!("lower: {error}")) - .into_iter() - .next() - .unwrap_or_else(|| panic!("the fixture declares a job")) -} - -struct Harness { - runs: Runs, - nodes: Nodes, - address: String, -} - -fn origin() -> Origin { - Origin::new( - "syncode/meta".to_owned(), - "a-commit".to_owned(), - "refs/heads/main".to_owned(), - "push".to_owned(), - ".gitea/workflows/ci.yml".to_owned(), - ) - .with_delivery(Some(format!("fixture-{}", uuid::Uuid::new_v4()))) -} - -fn repository_origin(repository: &str) -> Origin { - Origin::new( - repository.to_owned(), - "a-commit".to_owned(), - "refs/heads/main".to_owned(), - "push".to_owned(), - ".forgejo/workflows/ci.yml".to_owned(), - ) - .with_delivery(Some(format!("fixture-{}", uuid::Uuid::new_v4()))) -} - -fn capabilities() -> Capabilities { - Capabilities { - architecture: "arm64".to_owned(), - operating_system: "linux".to_owned(), - container_runtime: "docker".to_owned(), - container_runtime_version: "28.6.1".to_owned(), - cores: 2, - memory_bytes: 8 * 1024 * 1024 * 1024, - labels: vec!["self-hosted".to_owned(), "linux".to_owned()], - } -} - -fn capacity() -> Capacity { - Capacity { - build_volume_free_bytes: 60 * 1024 * 1024 * 1024, - layer_store_bytes: 12 * 1024 * 1024 * 1024, - cache_volume_present: true, - cache_volume_total_bytes: 100, - cache_volume_used_bytes: 40, - cache_volume_path: "/var/cache/syncode".to_owned(), - cached_images: Vec::new(), - cached_actions: Vec::new(), - } -} - -fn hello(sequence: u64, node: String, credential: String, capacity: Capacity) -> NodeMessage { - NodeMessage { - sequence, - message_id: format!("node-message-{sequence}"), - idempotency_key: format!("node-message-{sequence}"), - body: Some(node_message::Body::Hello(Hello { - node, - credential, - capabilities: Some(capabilities()), - capacity: Some(capacity), - max_parallel: 1, - })), - } -} - -/// What a node does before it can be given anything: spend a token, then open -/// a session with the credential it got back. -async fn enrolled( - harness: &Harness, - sender: &mpsc::Sender, - inbound: &mut Streaming, -) -> TestResult<(String, String)> { - let token = harness.nodes.issue_token(Scope::Instance).await?; - sender - .send(NodeMessage { - sequence: 1, - message_id: "node-message-1".to_owned(), - idempotency_key: "node-message-1".to_owned(), - body: Some(node_message::Body::Enrol(Enrol { - token: token.secret().expose().to_owned(), - })), - }) - .await?; - let message = next(inbound).await?; - let Some(control_message::Body::Enrolled(enrolled)) = message.body else { - panic!("expected an identity, got {:?}", message.body); - }; - Ok((enrolled.node, enrolled.credential)) -} - -async fn start() -> TestResult { - let (projection_base, mut projections) = projection_sink().await?; - tokio::spawn(async move { while projections.recv().await.is_some() {} }); - start_with_projection(projection_base).await -} - -async fn start_with_projection(projection_base: url::Url) -> TestResult { - let runs = Runs::restored(Forgotten::default()).await?; - let nodes = Nodes::restored(Ephemeral::default()).await?; - let listener = TcpListener::bind("127.0.0.1:0").await?; - let address = format!("http://{}", listener.local_addr()?); - let served = runs.clone(); - let served_nodes = nodes.clone(); - - tokio::spawn(async move { - let _ = Server::builder() - .add_service(GeneratedServer::new(NodeSessionServer::new( - served, - served_nodes, - CapabilityAuthority::new("test-capability-key").expect("capability key"), - ArtifactTokenAuthority::new("test-artifact-key").expect("artifact key"), - "http://127.0.0.1:1/".parse().expect("artifact URL"), - ProjectionClient::new(projection_base, String::new()).expect("projection client"), - ))) - .serve_with_incoming(TcpListenerStream::new(listener)) - .await; - }); - - Ok(Harness { - runs, - nodes, - address, - }) -} - -/// Answers every projection push with 204 and hands back the bodies it saw, -/// in order. -async fn projection_sink() -> TestResult<(url::Url, mpsc::UnboundedReceiver)> { - let listener = TcpListener::bind("127.0.0.1:0").await?; - let base = url::Url::parse(&format!("http://{}/", listener.local_addr()?))?; - let (sender, receiver) = mpsc::unbounded_channel(); - - tokio::spawn(async move { - loop { - let Ok((mut stream, _)) = listener.accept().await else { - return; - }; - let mut buffer = vec![0_u8; 8192]; - let read = tokio::io::AsyncReadExt::read(&mut stream, &mut buffer) - .await - .unwrap_or(0); - let request = String::from_utf8_lossy(&buffer[..read]).to_string(); - let body = request - .split_once("\r\n\r\n") - .map(|(_, body)| body.to_owned()) - .unwrap_or_default(); - let _ = sender.send(body); - let response = "HTTP/1.1 204 X\r\ncontent-length: 0\r\nconnection: close\r\n\r\n"; - let _ = tokio::io::AsyncWriteExt::write_all(&mut stream, response.as_bytes()).await; - let _ = tokio::io::AsyncWriteExt::shutdown(&mut stream).await; - } - }); - - Ok((base, receiver)) -} - -async fn wait_until_offline(harness: &Harness, node: syncode_control_runs::NodeId) -> TestResult { - for _ in 0..100 { - if harness.nodes.lifecycle(node).await? == Lifecycle::Offline { - return Ok(()); - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - Err("the closed node session did not become offline".into()) -} - -async fn next(stream: &mut Streaming) -> TestResult { - Ok(stream - .next() - .await - .ok_or("the control plane closed the stream")??) -} - -#[tokio::test] -async fn node_is_handed_the_plan_the_control_plane_compiled() -> TestResult { - let harness = start().await?; - let plan = VersionedPlan::new(compile()); - let run = harness - .runs - .queue(JobId::fresh(), origin(), serde_json::to_vec(&plan)?) - .await?; - - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - - let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; - sender - .send(NodeMessage { - sequence: 2, - message_id: "node-message-2".to_owned(), - idempotency_key: "node-message-2".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node: node.clone(), - credential, - capabilities: Some(capabilities()), - capacity: Some(capacity()), - max_parallel: 1, - })), - }) - .await?; - - let welcome = next(&mut inbound).await?; - assert!(matches!( - welcome.body, - Some(control_message::Body::Welcome(_)) - )); - assert_eq!(welcome.sequence, 2, "the stream numbers what it sends"); - - let assigned = next(&mut inbound).await?; - let Some(control_message::Body::Assignment(assignment)) = assigned.body else { - panic!("expected an assignment, got {:?}", assigned.body); - }; - assert_eq!(assignment.run, run.to_string()); - assert_eq!(assigned.sequence, 3); - - let carried: VersionedPlan = serde_json::from_slice(&assignment.plan)?; - assert_eq!(carried.schema(), PlanSchemaVersion::CURRENT); - assert_eq!(carried, plan); - assert_eq!(carried.plan().job().key().as_ref(), "build"); - - let RunState::Assigned(lease) = harness.runs.state_of(run).await? else { - panic!("the run must know who holds it before the assignment leaves"); - }; - assert_eq!(lease.node().to_string(), node); - assert_eq!(lease.fence().get(), assignment.fence); - - sender - .send(progress( - &run.to_string(), - &assignment.job, - assignment.fence, - Conclusion::Success, - )) - .await?; - - let mut state = harness.runs.state_of(run).await?; - for _ in 0..50 { - if matches!(state, RunState::Finished(_)) { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - state = harness.runs.state_of(run).await?; - } - assert_eq!(state, RunState::Finished(Conclusion::Success)); - - Ok(()) -} - -/// The node's `actions_runtime_token` is only good for spending once the forge -/// has heard which node and fence hold the job it names — so that has to be -/// true by the time the assignment carrying the token reaches the node, not -/// just eventually. This harness never runs the periodic sweep, so a body can -/// only arrive here because dispatch pushed it itself. -#[tokio::test] -async fn dispatch_projects_the_assignment_before_the_node_can_act_on_it() -> TestResult { - let (projection_base, mut seen) = projection_sink().await?; - let harness = start_with_projection(projection_base).await?; - let plan = VersionedPlan::new(compile()); - // A projection is only meaningful once it can be traced back to the - // delivery that started the run, so only a delivery-bearing origin is - // ever pushed; that's exactly the shape a real webhook delivery has. - let origin = origin().with_delivery(Some("a-delivery".to_owned())); - let run = harness - .runs - .queue(JobId::fresh(), origin, serde_json::to_vec(&plan)?) - .await?; - - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - - let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; - sender - .send(NodeMessage { - sequence: 2, - message_id: "node-message-2".to_owned(), - idempotency_key: "node-message-2".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node: node.clone(), - credential, - capabilities: Some(capabilities()), - capacity: Some(capacity()), - max_parallel: 1, - })), - }) - .await?; - let _welcome = next(&mut inbound).await?; - - let assigned = next(&mut inbound).await?; - let Some(control_message::Body::Assignment(assignment)) = assigned.body else { - panic!("expected an assignment, got {:?}", assigned.body); - }; - - let pushed = tokio::time::timeout(Duration::from_secs(1), seen.recv()) - .await? - .ok_or("the projection endpoint was never called")?; - assert!( - pushed.contains(&run.to_string()), - "the pushed body must name the run the node was just handed: {pushed}" - ); - assert!( - pushed.contains(&format!("\"job\":\"{}\"", assignment.job)), - "the pushed body must name the assigned job: {pushed}" - ); - assert!( - pushed.contains("\"kind\":\"assigned\""), - "the forge must already see the job as assigned, not waiting: {pushed}" - ); - assert!( - pushed.contains(&format!("\"node\":\"{node}\"")), - "the forge must already know which node holds it: {pushed}" - ); - - Ok(()) -} - -#[tokio::test] -async fn cancellation_is_pushed_with_the_current_job_and_fence() -> TestResult { - let harness = start().await?; - let job = JobId::fresh(); - let run = harness - .runs - .queue( - job, - origin(), - serde_json::to_vec(&VersionedPlan::new(compile()))?, - ) - .await?; - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; - sender.send(hello(2, node, credential, capacity())).await?; - let _ = next(&mut inbound).await?; - let assignment_message = next(&mut inbound).await?; - let Some(control_message::Body::Assignment(assignment)) = assignment_message.body else { - return Err("expected an assignment".into()); - }; - - let dispatch = harness.runs.cancel_job(run, job).await?; - assert!(matches!( - dispatch, - syncode_control_runs::CancellationDispatch::Requested(_) - )); - let cancellation_message = - tokio::time::timeout(Duration::from_secs(1), next(&mut inbound)).await??; - let cancellation_sequence = cancellation_message.sequence; - let Some(control_message::Body::Cancel(cancellation)) = cancellation_message.body else { - return Err("expected a cancellation".into()); - }; - assert_eq!(cancellation.run, run.to_string()); - assert_eq!(cancellation.job, job.to_string()); - assert_eq!(cancellation.fence, assignment.fence); - - sender - .send(NodeMessage { - sequence: 3, - message_id: "cancel-ack".to_owned(), - idempotency_key: "cancel-ack".to_owned(), - body: Some(node_message::Body::Acknowledgement(Acknowledgement { - message_id: cancellation_message.message_id, - log_offset: 0, - acknowledged_outputs: Vec::new(), - conclusion: syncode_control_node::wire::Conclusion::Unspecified as i32, - })), - }) - .await?; - sender - .send(NodeMessage { - sequence: 4, - message_id: "heartbeat-after-cancel-ack".to_owned(), - idempotency_key: "heartbeat-after-cancel-ack".to_owned(), - body: Some(node_message::Body::Heartbeat(Heartbeat { - held: Vec::new(), - capacity: Some(capacity()), - })), - }) - .await?; - let heartbeat_acknowledgement = next(&mut inbound).await?; - assert_eq!( - heartbeat_acknowledgement.sequence, - cancellation_sequence + 1 - ); - assert!(matches!( - heartbeat_acknowledgement.body, - Some(control_message::Body::Acknowledgement(_)) - )); - - sender - .send(NodeMessage { - sequence: 5, - message_id: "cancelled-progress".to_owned(), - idempotency_key: "cancelled-progress".to_owned(), - body: Some(node_message::Body::Progress(JobProgress { - run: run.to_string(), - job: job.to_string(), - conclusion: syncode_control_node::wire::Conclusion::Cancelled as i32, - fence: assignment.fence, - outputs: Default::default(), - })), - }) - .await?; - - for _ in 0..50 { - if harness.runs.state_of(run).await? == RunState::Finished(Conclusion::Cancelled) { - return Ok(()); - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - Err("cancelled progress did not finish the run".into()) -} - -#[tokio::test] -async fn a_job_requirement_mismatch_does_not_refuse_the_node() -> TestResult { - let harness = start().await?; - let event = Event::new( - EventKind::Push, - syncode_workflow::GitReference::Branch("main".to_owned()), - Vec::new(), - ); - let triggered = trigger( - &harness.runs, - br#" -on: [push] -jobs: - build: - runs-on: [self-hosted, linux, x64] - steps: - - run: cargo test -"#, - &event, - &origin(), - &FIXTURE_ACTIONS, - ) - .await?; - let Triggered::Runs(runs) = triggered else { - return Err("workflow did not trigger".into()); - }; - let run = *runs.first().ok_or("trigger returned no run")?; - - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; - sender - .send(NodeMessage { - sequence: 2, - message_id: "typed-hello".to_owned(), - idempotency_key: "typed-hello".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node, - credential, - capabilities: Some(capabilities()), - capacity: Some(capacity()), - max_parallel: 1, - })), - }) - .await?; - - assert!(matches!( - next(&mut inbound).await?.body, - Some(control_message::Body::Welcome(_)) - )); - assert!( - tokio::time::timeout(Duration::from_millis(200), inbound.next()) - .await - .is_err(), - "a job requirement mismatch closed the healthy node session" - ); - assert_eq!(harness.runs.state_of(run).await?, RunState::Queued); - - harness - .runs - .queue( - JobId::fresh(), - origin(), - serde_json::to_vec(&VersionedPlan::new(compile()))?, - ) - .await?; - let assignment = tokio::time::timeout(Duration::from_secs(1), next(&mut inbound)).await??; - assert!(matches!( - assignment.body, - Some(control_message::Body::Assignment(_)) - )); - Ok(()) -} - -#[tokio::test] -async fn stream_dispatch_does_not_let_one_organization_monopolize_capacity() -> TestResult { - let harness = start().await?; - let plan = serde_json::to_vec(&VersionedPlan::new(compile()))?; - for _ in 0..3 { - harness - .runs - .queue( - JobId::fresh(), - repository_origin("backlog/project"), - plan.clone(), - ) - .await?; - } - harness - .runs - .queue(JobId::fresh(), repository_origin("neighbor/project"), plan) - .await?; - - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; - sender - .send(NodeMessage { - sequence: 2, - message_id: "fair-hello".to_owned(), - idempotency_key: "fair-hello".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node, - credential, - capabilities: Some(capabilities()), - capacity: Some(capacity()), - max_parallel: 2, - })), - }) - .await?; - let _ = next(&mut inbound).await?; - let first = next(&mut inbound).await?; - let second = next(&mut inbound).await?; - let Some(control_message::Body::Assignment(first)) = first.body else { - return Err("first message was not an assignment".into()); - }; - let Some(control_message::Body::Assignment(second)) = second.body else { - return Err("second message was not an assignment".into()); - }; - assert_eq!( - first - .origin - .ok_or("first assignment has no origin")? - .repository, - "backlog/project" - ); - assert_eq!( - second - .origin - .ok_or("second assignment has no origin")? - .repository, - "neighbor/project" - ); - Ok(()) -} - -#[tokio::test] -async fn stream_dispatch_selects_the_warmest_connected_node() -> TestResult { - let harness = start().await?; - let (cold_sender, cold_receiver) = mpsc::channel(8); - let mut cold_client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut cold_inbound = cold_client - .open(ReceiverStream::new(cold_receiver)) - .await? - .into_inner(); - let (cold_node, cold_credential) = enrolled(&harness, &cold_sender, &mut cold_inbound).await?; - let mut cold_capacity = capacity(); - cold_capacity.cache_volume_present = false; - cold_capacity.cached_images.clear(); - cold_capacity.cached_actions.clear(); - cold_sender - .send(hello(2, cold_node, cold_credential, cold_capacity)) - .await?; - assert!(matches!( - next(&mut cold_inbound).await?.body, - Some(control_message::Body::Welcome(_)) - )); - - let (warm_sender, warm_receiver) = mpsc::channel(8); - let mut warm_client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut warm_inbound = warm_client - .open(ReceiverStream::new(warm_receiver)) - .await? - .into_inner(); - let (warm_node, warm_credential) = enrolled(&harness, &warm_sender, &mut warm_inbound).await?; - let mut warm_capacity = capacity(); - warm_capacity.cached_images = vec!["postgres:18".to_owned()]; - warm_capacity.cached_actions = vec!["actions/checkout@v5".to_owned()]; - warm_sender - .send(hello(2, warm_node, warm_credential, warm_capacity)) - .await?; - assert!(matches!( - next(&mut warm_inbound).await?.body, - Some(control_message::Body::Welcome(_)) - )); - - harness - .runs - .queue_run( - origin(), - vec![ - QueuedJob::new( - JobId::fresh(), - "build".to_owned(), - Vec::new(), - MatrixPolicy::default(), - vec![1], - ) - .scheduled( - Priority::Normal, - Requirements::default().prefer( - ["postgres:18".to_owned()], - ["actions/checkout@v5".to_owned()], - ), - ), - ], - ) - .await?; - - let message = tokio::time::timeout(Duration::from_secs(2), next(&mut warm_inbound)).await??; - assert!(matches!( - message.body, - Some(control_message::Body::Assignment(_)) - )); - assert!( - tokio::time::timeout(Duration::from_millis(200), next(&mut cold_inbound)) - .await - .is_err() - ); - Ok(()) -} - -#[tokio::test] -async fn a_node_receives_only_its_declared_number_of_concurrent_jobs() -> TestResult { - let harness = start().await?; - let plan = serde_json::to_vec(&VersionedPlan::new(compile()))?; - harness - .runs - .queue(JobId::fresh(), origin(), plan.clone()) - .await?; - harness - .runs - .queue(JobId::fresh(), origin(), plan.clone()) - .await?; - - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; - sender - .send(NodeMessage { - sequence: 2, - message_id: "node-message-2".to_owned(), - idempotency_key: "node-message-2".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node: node.clone(), - credential, - capabilities: Some(capabilities()), - capacity: Some(capacity()), - max_parallel: 2, - })), - }) - .await?; - - assert!(matches!( - next(&mut inbound).await?.body, - Some(control_message::Body::Welcome(_)) - )); - let first = next(&mut inbound).await?; - let second = next(&mut inbound).await?; - let Some(control_message::Body::Assignment(first)) = first.body else { - panic!("expected the first assignment"); - }; - let Some(control_message::Body::Assignment(second)) = second.body else { - panic!("expected the second assignment"); - }; - - harness.runs.queue(JobId::fresh(), origin(), plan).await?; - assert!( - tokio::time::timeout(Duration::from_millis(100), inbound.next()) - .await - .is_err(), - "a full node received a third concurrent job" - ); - - let completed = progress(&first.run, &first.job, first.fence, Conclusion::Success); - sender.send(completed.clone()).await?; - sender.send(completed).await?; - let replacement = next(&mut inbound).await?; - assert!(matches!( - replacement.body, - Some(control_message::Body::Assignment(_)) - )); - - sender - .send(NodeMessage { - sequence: 4, - message_id: "node-message-4".to_owned(), - idempotency_key: "node-message-4".to_owned(), - body: Some(node_message::Body::Progress(JobProgress { - run: second.run, - job: second.job, - conclusion: syncode_control_node::wire::Conclusion::Success.into(), - fence: second.fence, - outputs: Default::default(), - })), - }) - .await?; - Ok(()) -} - -#[tokio::test] -async fn an_out_of_order_node_message_closes_the_runtime_session() -> TestResult { - let harness = start().await?; - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; - sender - .send(NodeMessage { - sequence: 3, - message_id: "node-message-3".to_owned(), - idempotency_key: "node-message-3".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node, - credential, - capabilities: Some(capabilities()), - capacity: Some(capacity()), - max_parallel: 1, - })), - }) - .await?; - - let error = next(&mut inbound) - .await - .expect_err("the skipped sequence must close the stream"); - assert!(error.to_string().contains("while 2 was required")); - Ok(()) -} - -#[tokio::test] -async fn work_queued_after_hello_is_pushed_without_another_heartbeat() -> TestResult { - let harness = start().await?; - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; - sender - .send(NodeMessage { - sequence: 2, - message_id: "node-message-2".to_owned(), - idempotency_key: "node-message-2".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node, - credential, - capabilities: Some(capabilities()), - capacity: Some(capacity()), - max_parallel: 1, - })), - }) - .await?; - assert!(matches!( - next(&mut inbound).await?.body, - Some(control_message::Body::Welcome(_)) - )); - - harness - .runs - .queue( - JobId::fresh(), - origin(), - serde_json::to_vec(&VersionedPlan::new(compile()))?, - ) - .await?; - let pushed = tokio::time::timeout(Duration::from_secs(1), next(&mut inbound)).await??; - assert!(matches!( - pushed.body, - Some(control_message::Body::Assignment(_)) - )); - Ok(()) -} - -/// A node that keeps beating is handed its next credential before the one it -/// holds runs out, and the new one is what opens the session after that. Without -/// this a node works for exactly one credential term and is then locked out of -/// the control plane for good, with no way back but a fresh enrolment token. -#[tokio::test] -async fn a_credential_near_expiry_is_replaced_over_the_open_session() -> TestResult { - let harness = start().await?; - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - - let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; - sender - .send(NodeMessage { - sequence: 2, - message_id: "node-message-2".to_owned(), - idempotency_key: "node-message-2".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node: node.clone(), - credential: credential.clone(), - capabilities: Some(capabilities()), - capacity: Some(capacity()), - max_parallel: 1, - })), - }) - .await?; - - let welcome = next(&mut inbound).await?; - assert!( - matches!(welcome.body, Some(control_message::Body::Welcome(_))), - "a fresh credential is not renewed, got {:?}", - welcome.body - ); - - // Age the credential by issuing it as though the term had nearly run out. - // Sixty seconds of life left is inside the renewal margin, so the next thing - // the node says has to be answered with a replacement. - let issued = SystemTime::now() - CREDENTIAL_TERM + Duration::from_secs(60); - let ageing = harness.nodes.rotate(node.parse()?, issued).await?; - sender - .send(NodeMessage { - sequence: 3, - message_id: "node-message-3".to_owned(), - idempotency_key: "node-message-3".to_owned(), - body: Some(node_message::Body::Heartbeat(Heartbeat { - held: Vec::new(), - capacity: Some(capacity()), - })), - }) - .await?; - - let answer = next(&mut inbound).await?; - let Some(control_message::Body::Rotated(rotated)) = answer.body else { - panic!("expected a replacement credential, got {:?}", answer.body); - }; - assert_ne!( - rotated.credential, - ageing.secret().expose(), - "a rotation that hands back the same credential renews nothing" - ); - assert_ne!(rotated.credential, credential); - let acknowledged = next(&mut inbound).await?; - assert!(matches!( - acknowledged.body, - Some(control_message::Body::Acknowledgement(_)) - )); - - sender - .send(NodeMessage { - sequence: 4, - message_id: "node-message-4".to_owned(), - idempotency_key: "node-message-4".to_owned(), - body: Some(node_message::Body::Heartbeat(Heartbeat { - held: Vec::new(), - capacity: Some(capacity()), - })), - }) - .await?; - let repeated = next(&mut inbound).await?; - let Some(control_message::Body::Rotated(repeated)) = repeated.body else { - panic!("expected the unconfirmed credential again"); - }; - assert_eq!(repeated.credential, rotated.credential); - - drop(sender); - drop(inbound); - wait_until_offline(&harness, node.parse()?).await?; - - // The point of the exercise: the credential the node was pushed is the one - // that gets it back in, and the one it replaced does not. - let (fresh_sender, fresh_receiver) = mpsc::channel(8); - let mut reopened = client - .open(ReceiverStream::new(fresh_receiver)) - .await? - .into_inner(); - fresh_sender - .send(NodeMessage { - sequence: 1, - message_id: "node-message-1".to_owned(), - idempotency_key: "node-message-1".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node: node.clone(), - credential: rotated.credential, - capabilities: Some(capabilities()), - capacity: Some(capacity()), - max_parallel: 1, - })), - }) - .await?; - assert!(matches!( - next(&mut reopened).await?.body, - Some(control_message::Body::Welcome(_)) - )); - - let (stale_sender, stale_receiver) = mpsc::channel(8); - let mut refused = client - .open(ReceiverStream::new(stale_receiver)) - .await? - .into_inner(); - stale_sender - .send(NodeMessage { - sequence: 1, - message_id: "node-message-1".to_owned(), - idempotency_key: "node-message-1".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node, - credential, - capabilities: Some(capabilities()), - capacity: Some(capacity()), - max_parallel: 1, - })), - }) - .await?; - let error = next(&mut refused) - .await - .expect_err("the credential that was replaced must stop working"); - assert!( - error.to_string().contains("not current"), - "unexpected error: {error}" - ); - - Ok(()) -} - -/// An identity handed out to something that never arrived has to settle, or the -/// sweep reports it as newly gone every fifteen seconds forever and the log -/// stops being worth reading. -#[tokio::test] -async fn a_node_that_enrolled_and_never_spoke_is_reported_gone_once() -> TestResult { - let harness = start().await?; - let token = harness.nodes.issue_token(Scope::Instance).await?; - let now = SystemTime::now(); - let (node, _) = harness.nodes.enrol(token.secret().expose(), now).await?; - - let deadline = Duration::from_secs(90); - let late = now + Duration::from_secs(600); - assert_eq!( - harness.nodes.sweep_silent(deadline, late).await?, - vec![node] - ); - assert!( - harness.nodes.sweep_silent(deadline, late).await?.is_empty(), - "a node already marked offline is not gone again" - ); - - Ok(()) -} - -fn progress(run: &str, job: &str, fence: u64, conclusion: Conclusion) -> NodeMessage { - let conclusion = match conclusion { - Conclusion::Success => syncode_control_node::wire::Conclusion::Success, - Conclusion::Failure => syncode_control_node::wire::Conclusion::Failure, - Conclusion::Cancelled => syncode_control_node::wire::Conclusion::Cancelled, - Conclusion::Skipped => syncode_control_node::wire::Conclusion::Skipped, - }; - NodeMessage { - sequence: 3, - message_id: "node-message-3".to_owned(), - idempotency_key: "node-message-3".to_owned(), - body: Some(node_message::Body::Progress(JobProgress { - run: run.to_owned(), - job: job.to_owned(), - conclusion: conclusion.into(), - fence, - outputs: Default::default(), - })), - } -} - -#[tokio::test] -async fn report_stating_a_grant_that_is_not_current_is_refused() -> TestResult { - let harness = start().await?; - let plan = VersionedPlan::new(compile()); - let run = harness - .runs - .queue(JobId::fresh(), origin(), serde_json::to_vec(&plan)?) - .await?; - - let holder = syncode_control_runs::NodeId::fresh(); - let assignment = harness - .runs - .take_next(holder) - .await? - .ok_or("nothing was queued")?; - - let error = harness - .runs - .finished( - run, - holder, - Fence::from(assignment.fence().get() + 1), - Conclusion::Success, - ) - .await - .expect_err("a grant nobody was given must not conclude a run"); - - assert!( - error.to_string().contains("not the current grant"), - "unexpected error: {error}" - ); - assert!(matches!( - harness.runs.state_of(run).await?, - RunState::Assigned(_) - )); - - Ok(()) -} - -#[tokio::test] -async fn a_session_without_a_credential_is_refused() -> TestResult { - let harness = start().await?; - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - - let (node, _) = enrolled(&harness, &sender, &mut inbound).await?; - sender - .send(NodeMessage { - sequence: 2, - message_id: "node-message-2".to_owned(), - idempotency_key: "node-message-2".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node, - credential: "not the one it was given".to_owned(), - capabilities: Some(capabilities()), - capacity: Some(capacity()), - max_parallel: 1, - })), - }) - .await?; - - let error = next(&mut inbound) - .await - .expect_err("a wrong credential must end the session"); - assert!( - error.to_string().contains("not current"), - "unexpected error: {error}" - ); - - Ok(()) -} - -#[tokio::test] -async fn a_node_that_does_not_say_how_much_room_it_has_is_refused() -> TestResult { - let harness = start().await?; - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - - let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; - sender - .send(NodeMessage { - sequence: 2, - message_id: "node-message-2".to_owned(), - idempotency_key: "node-message-2".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node, - credential, - capabilities: Some(capabilities()), - capacity: None, - max_parallel: 1, - })), - }) - .await?; - - let error = next(&mut inbound) - .await - .expect_err("a node without capacity must not open a session"); - assert!( - error.to_string().contains("how much room"), - "unexpected error: {error}" - ); - - Ok(()) -} - -#[tokio::test] -async fn a_failed_capacity_refresh_pauses_work_without_closing_the_session() -> TestResult { - let harness = start().await?; - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; - sender - .send(NodeMessage { - sequence: 2, - message_id: "node-message-2".to_owned(), - idempotency_key: "node-message-2".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node: node.clone(), - credential, - capabilities: Some(capabilities()), - capacity: Some(capacity()), - max_parallel: 1, - })), - }) - .await?; - assert!(matches!( - next(&mut inbound).await?.body, - Some(control_message::Body::Welcome(_)) - )); - - sender - .send(NodeMessage { - sequence: 3, - message_id: "node-message-3".to_owned(), - idempotency_key: "node-message-3".to_owned(), - body: Some(node_message::Body::Heartbeat(Heartbeat { - held: Vec::new(), - capacity: None, - })), - }) - .await?; - let node_id = node.parse()?; - for _ in 0..50 { - if !harness.nodes.employable(node_id).await? { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - assert!(!harness.nodes.employable(node_id).await?); - assert!(matches!( - next(&mut inbound).await?.body, - Some(control_message::Body::Acknowledgement(_)) - )); - harness - .runs - .queue( - JobId::fresh(), - origin(), - serde_json::to_vec(&VersionedPlan::new(compile()))?, - ) - .await?; - - assert!( - tokio::time::timeout(Duration::from_millis(100), next(&mut inbound)) - .await - .is_err(), - "an unmeasured node must stay connected without receiving work" - ); - - sender - .send(NodeMessage { - sequence: 4, - message_id: "node-message-4".to_owned(), - idempotency_key: "node-message-4".to_owned(), - body: Some(node_message::Body::Heartbeat(Heartbeat { - held: Vec::new(), - capacity: Some(capacity()), - })), - }) - .await?; - let answer = next(&mut inbound).await?; - assert!( - matches!(answer.body, Some(control_message::Body::Assignment(_))), - "a recovered capacity measurement must resume work" - ); - Ok(()) -} - -#[tokio::test] -async fn a_node_short_of_room_is_told_why_instead_of_being_given_work() -> TestResult { - let harness = start().await?; - let plan = VersionedPlan::new(compile()); - harness - .runs - .queue(JobId::fresh(), origin(), serde_json::to_vec(&plan)?) - .await?; - - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - - let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; - let cramped = Capacity { - build_volume_free_bytes: 1024, - ..capacity() - }; - sender - .send(NodeMessage { - sequence: 2, - message_id: "node-message-2".to_owned(), - idempotency_key: "node-message-2".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node, - credential, - capabilities: Some(capabilities()), - capacity: Some(cramped), - max_parallel: 1, - })), - }) - .await?; - - let welcome = next(&mut inbound).await?; - assert!(matches!( - welcome.body, - Some(control_message::Body::Welcome(_)) - )); - - let answer = next(&mut inbound).await?; - let Some(control_message::Body::Refused(refused)) = answer.body else { - panic!("expected a stated refusal, got {:?}", answer.body); - }; - assert!( - refused.reason.contains("below the"), - "unexpected reason: {}", - refused.reason - ); - - Ok(()) -} - -#[tokio::test] -async fn an_applied_message_is_deduplicated_after_reconnect() -> TestResult { - let harness = start().await?; - let (first_sender, first_receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut first = client - .open(ReceiverStream::new(first_receiver)) - .await? - .into_inner(); - let (node, credential) = enrolled(&harness, &first_sender, &mut first).await?; - first_sender - .send(NodeMessage { - sequence: 2, - message_id: "first-hello".to_owned(), - idempotency_key: "first-hello".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node: node.clone(), - credential: credential.clone(), - capabilities: Some(capabilities()), - capacity: Some(capacity()), - max_parallel: 1, - })), - }) - .await?; - let _ = next(&mut first).await?; - let repeated = NodeMessage { - sequence: 3, - message_id: "persistent-heartbeat".to_owned(), - idempotency_key: "persistent-heartbeat".to_owned(), - body: Some(node_message::Body::Heartbeat(Heartbeat { - held: Vec::new(), - capacity: None, - })), - }; - first_sender.send(repeated.clone()).await?; - let node_id = node.parse()?; - for _ in 0..50 { - if !harness.nodes.employable(node_id).await? { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - assert!(!harness.nodes.employable(node_id).await?); - - drop(first_sender); - drop(first); - wait_until_offline(&harness, node_id).await?; - - let (second_sender, second_receiver) = mpsc::channel(8); - let mut second_client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut second = second_client - .open(ReceiverStream::new(second_receiver)) - .await? - .into_inner(); - second_sender - .send(NodeMessage { - sequence: 1, - message_id: "second-hello".to_owned(), - idempotency_key: "second-hello".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node, - credential, - capabilities: Some(capabilities()), - capacity: Some(capacity()), - max_parallel: 1, - })), - }) - .await?; - let _ = next(&mut second).await?; - assert!(harness.nodes.employable(node_id).await?); - - second_sender - .send(NodeMessage { - sequence: 2, - ..repeated - }) - .await?; - tokio::time::sleep(Duration::from_millis(50)).await; - assert!( - harness.nodes.employable(node_id).await?, - "the duplicate must not clear capacity again" - ); - - second_sender - .send(NodeMessage { - sequence: 3, - message_id: "new-heartbeat".to_owned(), - idempotency_key: "new-heartbeat".to_owned(), - body: Some(node_message::Body::Heartbeat(Heartbeat { - held: Vec::new(), - capacity: None, - })), - }) - .await?; - for _ in 0..50 { - if !harness.nodes.employable(node_id).await? { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - assert!(!harness.nodes.employable(node_id).await?); - Ok(()) -} - -#[tokio::test] -async fn an_unacknowledged_assignment_is_retried_on_the_open_session() -> TestResult { - let harness = start().await?; - harness - .runs - .queue( - JobId::fresh(), - origin(), - serde_json::to_vec(&VersionedPlan::new(compile()))?, - ) - .await?; - - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; - sender - .send(NodeMessage { - sequence: 2, - message_id: "hello".to_owned(), - idempotency_key: "hello".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node: node.clone(), - credential, - capabilities: Some(capabilities()), - capacity: Some(capacity()), - max_parallel: 1, - })), - }) - .await?; - let _ = next(&mut inbound).await?; - let assignment = next(&mut inbound).await?; - assert!(matches!( - assignment.body, - Some(control_message::Body::Assignment(_)) - )); - - let retried = tokio::time::timeout(Duration::from_secs(3), next(&mut inbound)).await??; - assert_eq!(retried, assignment); - - sender - .send(NodeMessage { - sequence: 3, - message_id: "assignment-ack".to_owned(), - idempotency_key: "assignment-ack".to_owned(), - body: Some(node_message::Body::Acknowledgement(Acknowledgement { - message_id: assignment.message_id.clone(), - log_offset: 0, - acknowledged_outputs: Vec::new(), - conclusion: 0, - })), - }) - .await?; - let node = node.parse()?; - for _ in 0..50 { - if harness.nodes.outbound_pending(node).await?.is_empty() { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - assert!(harness.nodes.outbound_pending(node).await?.is_empty()); - - let acknowledgement = Acknowledgement { - message_id: assignment.message_id, - log_offset: 0, - acknowledged_outputs: Vec::new(), - conclusion: 0, - }; - sender - .send(NodeMessage { - sequence: 4, - message_id: "retried-assignment-ack".to_owned(), - idempotency_key: "assignment-ack".to_owned(), - body: Some(node_message::Body::Acknowledgement(acknowledgement)), - }) - .await?; - sender - .send(NodeMessage { - sequence: 5, - message_id: "heartbeat-after-retried-ack".to_owned(), - idempotency_key: "heartbeat-after-retried-ack".to_owned(), - body: Some(node_message::Body::Heartbeat(Heartbeat { - held: Vec::new(), - capacity: Some(capacity()), - })), - }) - .await?; - assert!(matches!( - next(&mut inbound).await?.body, - Some(control_message::Body::Acknowledgement(_)) - )); - Ok(()) -} - -#[tokio::test] -async fn reconnect_does_not_restore_capacity_while_a_lease_is_held() -> TestResult { - let harness = start().await?; - let plan = serde_json::to_vec(&VersionedPlan::new(compile()))?; - let first_run = harness - .runs - .queue(JobId::fresh(), origin(), plan.clone()) - .await?; - let second_run = harness.runs.queue(JobId::fresh(), origin(), plan).await?; - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; - sender - .send(hello(2, node.clone(), credential.clone(), capacity())) - .await?; - let _ = next(&mut inbound).await?; - let assigned = next(&mut inbound).await?; - let Some(control_message::Body::Assignment(assignment)) = assigned.body.clone() else { - return Err("expected the first assignment".into()); - }; - assert_eq!(assignment.run, first_run.to_string()); - sender - .send(NodeMessage { - sequence: 3, - message_id: "assignment-ack".to_owned(), - idempotency_key: "assignment-ack".to_owned(), - body: Some(node_message::Body::Acknowledgement(Acknowledgement { - message_id: assigned.message_id, - log_offset: 0, - acknowledged_outputs: Vec::new(), - conclusion: 0, - })), - }) - .await?; - let node_id = node.parse()?; - for _ in 0..50 { - if harness.nodes.outbound_pending(node_id).await?.is_empty() { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - drop(sender); - drop(inbound); - wait_until_offline(&harness, node_id).await?; - - let (sender, receiver) = mpsc::channel(8); - let mut reopened = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - sender.send(hello(1, node, credential, capacity())).await?; - let _ = next(&mut reopened).await?; - assert!( - tokio::time::timeout(Duration::from_millis(200), next(&mut reopened)) - .await - .is_err(), - "reconnect restored a slot still occupied by the first lease" - ); - - harness - .runs - .finished_job( - first_run, - assignment.job.parse()?, - node_id, - Fence::from(assignment.fence), - Conclusion::Success, - ) - .await?; - let next_assignment = - tokio::time::timeout(Duration::from_secs(1), next(&mut reopened)).await??; - let Some(control_message::Body::Assignment(assignment)) = next_assignment.body else { - return Err("expected the second assignment".into()); - }; - assert_eq!(assignment.run, second_run.to_string()); - Ok(()) -} - -#[tokio::test] -async fn a_silent_open_session_is_refused_after_its_lease_expires() -> TestResult { - let harness = start().await?; - let plan = serde_json::to_vec(&VersionedPlan::new(compile()))?; - let run = harness.runs.queue(JobId::fresh(), origin(), plan).await?; - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.address.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; - sender - .send(hello(2, node.clone(), credential, capacity())) - .await?; - let _ = next(&mut inbound).await?; - let assignment = next(&mut inbound).await?; - assert!(matches!( - assignment.body, - Some(control_message::Body::Assignment(_)) - )); - - let node_id = node.parse()?; - let late = SystemTime::now() + Duration::from_secs(600); - assert_eq!( - harness - .nodes - .sweep_silent(Duration::from_secs(90), late) - .await?, - vec![node_id] - ); - assert_eq!(harness.nodes.lifecycle(node_id).await?, Lifecycle::Offline); - assert_eq!(harness.runs.reclaim_expired(late).await?, vec![run]); - - let refused = tokio::time::timeout(Duration::from_secs(1), next(&mut inbound)).await??; - assert!(matches!( - refused.body, - Some(control_message::Body::Refused(_)) - )); - assert_eq!(harness.runs.state_of(run).await?, RunState::Queued); - Ok(()) -} +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] + +#[path = "support/actions.rs"] +mod actions; + +use std::error::Error; +use std::time::{Duration, SystemTime}; + +use syncode_control::trigger::{Triggered, trigger}; +use syncode_control_node::wire::{ + Acknowledgement, Capabilities, Capacity, ControlMessage, Enrol, Heartbeat, Hello, JobProgress, + NodeMessage, control_message, node_message, +}; +use syncode_control_node::{ + ArtifactTokenAuthority, CapabilityAuthority, GeneratedServer, NodeSessionClient, + NodeSessionServer, ProjectionClient, +}; +use syncode_control_nodes::{CREDENTIAL_TERM, Ephemeral, Lifecycle, Nodes, Scope}; +use syncode_control_runs::{ + Conclusion, Fence, Forgotten, JobId, MatrixPolicy, Origin, Priority, QueuedJob, Requirements, + RunState, Runs, +}; +use syncode_workflow::{ + Event, EventKind, ExecutionPlan, PlanSchemaVersion, VersionedPlan, WorkflowCompiler, + WorkflowDialect, WorkflowSource, +}; +use syncode_workflow_github_actions::compiler::GithubActionsCompiler; +use syncode_workflow_github_actions::expression::ExpressionProgram; +use tokio::net::TcpListener; +use tokio::sync::mpsc; +use tokio_stream::StreamExt; +use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream}; +use tonic::Streaming; +use tonic::transport::Server; + +use actions::FIXTURE_ACTIONS; + +type TestResult = Result>; + +const WORKFLOW: &str = r#" +name: CI +jobs: + build: + runs-on: [self-hosted, linux] + steps: + - run: echo "the plan came from the control plane" +"#; + +fn compile() -> ExecutionPlan { + let hir = GithubActionsCompiler + .compile(&WorkflowSource::new( + WorkflowDialect::GitHubActions, + WORKFLOW.as_bytes().to_vec(), + )) + .unwrap_or_else(|error| panic!("compile: {error}")); + syncode_workflow::plans(hir) + .unwrap_or_else(|error| panic!("lower: {error}")) + .into_iter() + .next() + .unwrap_or_else(|| panic!("the fixture declares a job")) +} + +struct Harness { + runs: Runs, + nodes: Nodes, + address: String, +} + +fn origin() -> Origin { + Origin::new( + "syncode/meta".to_owned(), + "a-commit".to_owned(), + "refs/heads/main".to_owned(), + "push".to_owned(), + ".gitea/workflows/ci.yml".to_owned(), + ) + .with_delivery(Some(format!("fixture-{}", uuid::Uuid::new_v4()))) +} + +fn repository_origin(repository: &str) -> Origin { + Origin::new( + repository.to_owned(), + "a-commit".to_owned(), + "refs/heads/main".to_owned(), + "push".to_owned(), + ".forgejo/workflows/ci.yml".to_owned(), + ) + .with_delivery(Some(format!("fixture-{}", uuid::Uuid::new_v4()))) +} + +fn capabilities() -> Capabilities { + Capabilities { + architecture: "arm64".to_owned(), + operating_system: "linux".to_owned(), + container_runtime: "docker".to_owned(), + container_runtime_version: "28.6.1".to_owned(), + cores: 2, + memory_bytes: 8 * 1024 * 1024 * 1024, + labels: vec!["self-hosted".to_owned(), "linux".to_owned()], + } +} + +fn capacity() -> Capacity { + Capacity { + build_volume_free_bytes: 60 * 1024 * 1024 * 1024, + layer_store_bytes: 12 * 1024 * 1024 * 1024, + cache_volume_present: true, + cache_volume_total_bytes: 100, + cache_volume_used_bytes: 40, + cache_volume_path: "/var/cache/syncode".to_owned(), + cached_images: Vec::new(), + cached_actions: Vec::new(), + } +} + +fn hello(sequence: u64, node: String, credential: String, capacity: Capacity) -> NodeMessage { + NodeMessage { + sequence, + message_id: format!("node-message-{sequence}"), + idempotency_key: format!("node-message-{sequence}"), + body: Some(node_message::Body::Hello(Hello { + node, + credential, + capabilities: Some(capabilities()), + capacity: Some(capacity), + max_parallel: 1, + })), + } +} + +/// What a node does before it can be given anything: spend a token, then open +/// a session with the credential it got back. +async fn enrolled( + harness: &Harness, + sender: &mpsc::Sender, + inbound: &mut Streaming, +) -> TestResult<(String, String)> { + let token = harness.nodes.issue_token(Scope::Instance).await?; + sender + .send(NodeMessage { + sequence: 1, + message_id: "node-message-1".to_owned(), + idempotency_key: "node-message-1".to_owned(), + body: Some(node_message::Body::Enrol(Enrol { + token: token.secret().expose().to_owned(), + })), + }) + .await?; + let message = next(inbound).await?; + let Some(control_message::Body::Enrolled(enrolled)) = message.body else { + panic!("expected an identity, got {:?}", message.body); + }; + Ok((enrolled.node, enrolled.credential)) +} + +async fn start() -> TestResult { + let (projection_base, mut projections) = projection_sink().await?; + tokio::spawn(async move { while projections.recv().await.is_some() {} }); + start_with_projection(projection_base).await +} + +async fn start_with_projection(projection_base: url::Url) -> TestResult { + let runs = Runs::restored(Forgotten::default()).await?; + let nodes = Nodes::restored(Ephemeral::default()).await?; + let listener = TcpListener::bind("127.0.0.1:0").await?; + let address = format!("http://{}", listener.local_addr()?); + let served = runs.clone(); + let served_nodes = nodes.clone(); + + tokio::spawn(async move { + let _ = Server::builder() + .add_service(GeneratedServer::new(NodeSessionServer::new( + served, + served_nodes, + CapabilityAuthority::new("test-capability-key").expect("capability key"), + ArtifactTokenAuthority::new("test-artifact-key").expect("artifact key"), + "http://127.0.0.1:1/".parse().expect("artifact URL"), + ProjectionClient::new( + projection_base, + String::new(), + "http://127.0.0.1:1".to_owned(), + String::new(), + ) + .expect("projection client"), + ))) + .serve_with_incoming(TcpListenerStream::new(listener)) + .await; + }); + + Ok(Harness { + runs, + nodes, + address, + }) +} + +/// Answers every projection push with 204 and hands back the bodies it saw, +/// in order. +async fn projection_sink() -> TestResult<(url::Url, mpsc::UnboundedReceiver)> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let base = url::Url::parse(&format!("http://{}/", listener.local_addr()?))?; + let (sender, receiver) = mpsc::unbounded_channel(); + + tokio::spawn(async move { + loop { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + let mut buffer = vec![0_u8; 8192]; + let read = tokio::io::AsyncReadExt::read(&mut stream, &mut buffer) + .await + .unwrap_or(0); + let request = String::from_utf8_lossy(&buffer[..read]).to_string(); + let body = request + .split_once("\r\n\r\n") + .map(|(_, body)| body.to_owned()) + .unwrap_or_default(); + let _ = sender.send(body); + let response = "HTTP/1.1 204 X\r\ncontent-length: 0\r\nconnection: close\r\n\r\n"; + let _ = tokio::io::AsyncWriteExt::write_all(&mut stream, response.as_bytes()).await; + let _ = tokio::io::AsyncWriteExt::shutdown(&mut stream).await; + } + }); + + Ok((base, receiver)) +} + +async fn wait_until_offline(harness: &Harness, node: syncode_control_runs::NodeId) -> TestResult { + for _ in 0..100 { + if harness.nodes.lifecycle(node).await? == Lifecycle::Offline { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + Err("the closed node session did not become offline".into()) +} + +async fn next(stream: &mut Streaming) -> TestResult { + Ok(stream + .next() + .await + .ok_or("the control plane closed the stream")??) +} + +#[tokio::test] +async fn node_is_handed_the_plan_the_control_plane_compiled() -> TestResult { + let harness = start().await?; + let plan = VersionedPlan::new(compile()); + let run = harness + .runs + .queue(JobId::fresh(), origin(), serde_json::to_vec(&plan)?) + .await?; + + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + + let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; + sender + .send(NodeMessage { + sequence: 2, + message_id: "node-message-2".to_owned(), + idempotency_key: "node-message-2".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node: node.clone(), + credential, + capabilities: Some(capabilities()), + capacity: Some(capacity()), + max_parallel: 1, + })), + }) + .await?; + + let welcome = next(&mut inbound).await?; + assert!(matches!( + welcome.body, + Some(control_message::Body::Welcome(_)) + )); + assert_eq!(welcome.sequence, 2, "the stream numbers what it sends"); + + let assigned = next(&mut inbound).await?; + let Some(control_message::Body::Assignment(assignment)) = assigned.body else { + panic!("expected an assignment, got {:?}", assigned.body); + }; + assert_eq!(assignment.run, run.to_string()); + assert_eq!(assigned.sequence, 3); + + let carried: VersionedPlan = serde_json::from_slice(&assignment.plan)?; + assert_eq!(carried.schema(), PlanSchemaVersion::CURRENT); + assert_eq!(carried, plan); + assert_eq!(carried.plan().job().key().as_ref(), "build"); + + let RunState::Assigned(lease) = harness.runs.state_of(run).await? else { + panic!("the run must know who holds it before the assignment leaves"); + }; + assert_eq!(lease.node().to_string(), node); + assert_eq!(lease.fence().get(), assignment.fence); + + sender + .send(progress( + &run.to_string(), + &assignment.job, + assignment.fence, + Conclusion::Success, + )) + .await?; + + let mut state = harness.runs.state_of(run).await?; + for _ in 0..50 { + if matches!(state, RunState::Finished(_)) { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + state = harness.runs.state_of(run).await?; + } + assert_eq!(state, RunState::Finished(Conclusion::Success)); + + Ok(()) +} + +/// The node's `actions_runtime_token` is only good for spending once the forge +/// has heard which node and fence hold the job it names — so that has to be +/// true by the time the assignment carrying the token reaches the node, not +/// just eventually. This harness never runs the periodic sweep, so a body can +/// only arrive here because dispatch pushed it itself. +#[tokio::test] +async fn dispatch_projects_the_assignment_before_the_node_can_act_on_it() -> TestResult { + let (projection_base, mut seen) = projection_sink().await?; + let harness = start_with_projection(projection_base).await?; + let plan = VersionedPlan::new(compile()); + // A projection is only meaningful once it can be traced back to the + // delivery that started the run, so only a delivery-bearing origin is + // ever pushed; that's exactly the shape a real webhook delivery has. + let origin = origin().with_delivery(Some("a-delivery".to_owned())); + let run = harness + .runs + .queue(JobId::fresh(), origin, serde_json::to_vec(&plan)?) + .await?; + + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + + let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; + sender + .send(NodeMessage { + sequence: 2, + message_id: "node-message-2".to_owned(), + idempotency_key: "node-message-2".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node: node.clone(), + credential, + capabilities: Some(capabilities()), + capacity: Some(capacity()), + max_parallel: 1, + })), + }) + .await?; + let _welcome = next(&mut inbound).await?; + + let assigned = next(&mut inbound).await?; + let Some(control_message::Body::Assignment(assignment)) = assigned.body else { + panic!("expected an assignment, got {:?}", assigned.body); + }; + + let pushed = tokio::time::timeout(Duration::from_secs(1), seen.recv()) + .await? + .ok_or("the projection endpoint was never called")?; + assert!( + pushed.contains(&run.to_string()), + "the pushed body must name the run the node was just handed: {pushed}" + ); + assert!( + pushed.contains(&format!("\"job\":\"{}\"", assignment.job)), + "the pushed body must name the assigned job: {pushed}" + ); + assert!( + pushed.contains("\"kind\":\"assigned\""), + "the forge must already see the job as assigned, not waiting: {pushed}" + ); + assert!( + pushed.contains(&format!("\"node\":\"{node}\"")), + "the forge must already know which node holds it: {pushed}" + ); + + Ok(()) +} + +#[tokio::test] +async fn cancellation_is_pushed_with_the_current_job_and_fence() -> TestResult { + let harness = start().await?; + let job = JobId::fresh(); + let run = harness + .runs + .queue( + job, + origin(), + serde_json::to_vec(&VersionedPlan::new(compile()))?, + ) + .await?; + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; + sender.send(hello(2, node, credential, capacity())).await?; + let _ = next(&mut inbound).await?; + let assignment_message = next(&mut inbound).await?; + let Some(control_message::Body::Assignment(assignment)) = assignment_message.body else { + return Err("expected an assignment".into()); + }; + + let dispatch = harness.runs.cancel_job(run, job).await?; + assert!(matches!( + dispatch, + syncode_control_runs::CancellationDispatch::Requested(_) + )); + let cancellation_message = + tokio::time::timeout(Duration::from_secs(1), next(&mut inbound)).await??; + let cancellation_sequence = cancellation_message.sequence; + let Some(control_message::Body::Cancel(cancellation)) = cancellation_message.body else { + return Err("expected a cancellation".into()); + }; + assert_eq!(cancellation.run, run.to_string()); + assert_eq!(cancellation.job, job.to_string()); + assert_eq!(cancellation.fence, assignment.fence); + + sender + .send(NodeMessage { + sequence: 3, + message_id: "cancel-ack".to_owned(), + idempotency_key: "cancel-ack".to_owned(), + body: Some(node_message::Body::Acknowledgement(Acknowledgement { + message_id: cancellation_message.message_id, + log_offset: 0, + acknowledged_outputs: Vec::new(), + conclusion: syncode_control_node::wire::Conclusion::Unspecified as i32, + })), + }) + .await?; + sender + .send(NodeMessage { + sequence: 4, + message_id: "heartbeat-after-cancel-ack".to_owned(), + idempotency_key: "heartbeat-after-cancel-ack".to_owned(), + body: Some(node_message::Body::Heartbeat(Heartbeat { + held: Vec::new(), + capacity: Some(capacity()), + })), + }) + .await?; + let heartbeat_acknowledgement = next(&mut inbound).await?; + assert_eq!( + heartbeat_acknowledgement.sequence, + cancellation_sequence + 1 + ); + assert!(matches!( + heartbeat_acknowledgement.body, + Some(control_message::Body::Acknowledgement(_)) + )); + + sender + .send(NodeMessage { + sequence: 5, + message_id: "cancelled-progress".to_owned(), + idempotency_key: "cancelled-progress".to_owned(), + body: Some(node_message::Body::Progress(JobProgress { + run: run.to_string(), + job: job.to_string(), + conclusion: syncode_control_node::wire::Conclusion::Cancelled as i32, + fence: assignment.fence, + outputs: Default::default(), + })), + }) + .await?; + + for _ in 0..50 { + if harness.runs.state_of(run).await? == RunState::Finished(Conclusion::Cancelled) { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + Err("cancelled progress did not finish the run".into()) +} + +#[tokio::test] +async fn a_job_requirement_mismatch_does_not_refuse_the_node() -> TestResult { + let harness = start().await?; + let event = Event::new( + EventKind::Push, + syncode_workflow::GitReference::Branch("main".to_owned()), + Vec::new(), + ); + let triggered = trigger( + &harness.runs, + br#" +on: [push] +jobs: + build: + runs-on: [self-hosted, linux, x64] + steps: + - run: cargo test +"#, + &event, + &origin(), + &FIXTURE_ACTIONS, + ) + .await?; + let Triggered::Runs(runs) = triggered else { + return Err("workflow did not trigger".into()); + }; + let run = *runs.first().ok_or("trigger returned no run")?; + + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; + sender + .send(NodeMessage { + sequence: 2, + message_id: "typed-hello".to_owned(), + idempotency_key: "typed-hello".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node, + credential, + capabilities: Some(capabilities()), + capacity: Some(capacity()), + max_parallel: 1, + })), + }) + .await?; + + assert!(matches!( + next(&mut inbound).await?.body, + Some(control_message::Body::Welcome(_)) + )); + assert!( + tokio::time::timeout(Duration::from_millis(200), inbound.next()) + .await + .is_err(), + "a job requirement mismatch closed the healthy node session" + ); + assert_eq!(harness.runs.state_of(run).await?, RunState::Queued); + + harness + .runs + .queue( + JobId::fresh(), + origin(), + serde_json::to_vec(&VersionedPlan::new(compile()))?, + ) + .await?; + let assignment = tokio::time::timeout(Duration::from_secs(1), next(&mut inbound)).await??; + assert!(matches!( + assignment.body, + Some(control_message::Body::Assignment(_)) + )); + Ok(()) +} + +#[tokio::test] +async fn stream_dispatch_does_not_let_one_organization_monopolize_capacity() -> TestResult { + let harness = start().await?; + let plan = serde_json::to_vec(&VersionedPlan::new(compile()))?; + for _ in 0..3 { + harness + .runs + .queue( + JobId::fresh(), + repository_origin("backlog/project"), + plan.clone(), + ) + .await?; + } + harness + .runs + .queue(JobId::fresh(), repository_origin("neighbor/project"), plan) + .await?; + + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; + sender + .send(NodeMessage { + sequence: 2, + message_id: "fair-hello".to_owned(), + idempotency_key: "fair-hello".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node, + credential, + capabilities: Some(capabilities()), + capacity: Some(capacity()), + max_parallel: 2, + })), + }) + .await?; + let _ = next(&mut inbound).await?; + let first = next(&mut inbound).await?; + let second = next(&mut inbound).await?; + let Some(control_message::Body::Assignment(first)) = first.body else { + return Err("first message was not an assignment".into()); + }; + let Some(control_message::Body::Assignment(second)) = second.body else { + return Err("second message was not an assignment".into()); + }; + assert_eq!( + first + .origin + .ok_or("first assignment has no origin")? + .repository, + "backlog/project" + ); + assert_eq!( + second + .origin + .ok_or("second assignment has no origin")? + .repository, + "neighbor/project" + ); + Ok(()) +} + +#[tokio::test] +async fn stream_dispatch_selects_the_warmest_connected_node() -> TestResult { + let harness = start().await?; + let (cold_sender, cold_receiver) = mpsc::channel(8); + let mut cold_client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut cold_inbound = cold_client + .open(ReceiverStream::new(cold_receiver)) + .await? + .into_inner(); + let (cold_node, cold_credential) = enrolled(&harness, &cold_sender, &mut cold_inbound).await?; + let mut cold_capacity = capacity(); + cold_capacity.cache_volume_present = false; + cold_capacity.cached_images.clear(); + cold_capacity.cached_actions.clear(); + cold_sender + .send(hello(2, cold_node, cold_credential, cold_capacity)) + .await?; + assert!(matches!( + next(&mut cold_inbound).await?.body, + Some(control_message::Body::Welcome(_)) + )); + + let (warm_sender, warm_receiver) = mpsc::channel(8); + let mut warm_client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut warm_inbound = warm_client + .open(ReceiverStream::new(warm_receiver)) + .await? + .into_inner(); + let (warm_node, warm_credential) = enrolled(&harness, &warm_sender, &mut warm_inbound).await?; + let mut warm_capacity = capacity(); + warm_capacity.cached_images = vec!["postgres:18".to_owned()]; + warm_capacity.cached_actions = vec!["actions/checkout@v5".to_owned()]; + warm_sender + .send(hello(2, warm_node, warm_credential, warm_capacity)) + .await?; + assert!(matches!( + next(&mut warm_inbound).await?.body, + Some(control_message::Body::Welcome(_)) + )); + + harness + .runs + .queue_run( + origin(), + vec![ + QueuedJob::new( + JobId::fresh(), + "build".to_owned(), + Vec::new(), + MatrixPolicy::default(), + vec![1], + ) + .scheduled( + Priority::Normal, + Requirements::default().prefer( + ["postgres:18".to_owned()], + ["actions/checkout@v5".to_owned()], + ), + ), + ], + ) + .await?; + + let message = tokio::time::timeout(Duration::from_secs(2), next(&mut warm_inbound)).await??; + assert!(matches!( + message.body, + Some(control_message::Body::Assignment(_)) + )); + assert!( + tokio::time::timeout(Duration::from_millis(200), next(&mut cold_inbound)) + .await + .is_err() + ); + Ok(()) +} + +#[tokio::test] +async fn a_node_receives_only_its_declared_number_of_concurrent_jobs() -> TestResult { + let harness = start().await?; + let plan = serde_json::to_vec(&VersionedPlan::new(compile()))?; + harness + .runs + .queue(JobId::fresh(), origin(), plan.clone()) + .await?; + harness + .runs + .queue(JobId::fresh(), origin(), plan.clone()) + .await?; + + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; + sender + .send(NodeMessage { + sequence: 2, + message_id: "node-message-2".to_owned(), + idempotency_key: "node-message-2".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node: node.clone(), + credential, + capabilities: Some(capabilities()), + capacity: Some(capacity()), + max_parallel: 2, + })), + }) + .await?; + + assert!(matches!( + next(&mut inbound).await?.body, + Some(control_message::Body::Welcome(_)) + )); + let first = next(&mut inbound).await?; + let second = next(&mut inbound).await?; + let Some(control_message::Body::Assignment(first)) = first.body else { + panic!("expected the first assignment"); + }; + let Some(control_message::Body::Assignment(second)) = second.body else { + panic!("expected the second assignment"); + }; + + harness.runs.queue(JobId::fresh(), origin(), plan).await?; + assert!( + tokio::time::timeout(Duration::from_millis(100), inbound.next()) + .await + .is_err(), + "a full node received a third concurrent job" + ); + + let completed = progress(&first.run, &first.job, first.fence, Conclusion::Success); + sender.send(completed.clone()).await?; + sender.send(completed).await?; + let replacement = next(&mut inbound).await?; + assert!(matches!( + replacement.body, + Some(control_message::Body::Assignment(_)) + )); + + sender + .send(NodeMessage { + sequence: 4, + message_id: "node-message-4".to_owned(), + idempotency_key: "node-message-4".to_owned(), + body: Some(node_message::Body::Progress(JobProgress { + run: second.run, + job: second.job, + conclusion: syncode_control_node::wire::Conclusion::Success.into(), + fence: second.fence, + outputs: Default::default(), + })), + }) + .await?; + Ok(()) +} + +#[tokio::test] +async fn an_out_of_order_node_message_closes_the_runtime_session() -> TestResult { + let harness = start().await?; + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; + sender + .send(NodeMessage { + sequence: 3, + message_id: "node-message-3".to_owned(), + idempotency_key: "node-message-3".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node, + credential, + capabilities: Some(capabilities()), + capacity: Some(capacity()), + max_parallel: 1, + })), + }) + .await?; + + let error = next(&mut inbound) + .await + .expect_err("the skipped sequence must close the stream"); + assert!(error.to_string().contains("while 2 was required")); + Ok(()) +} + +#[tokio::test] +async fn work_queued_after_hello_is_pushed_without_another_heartbeat() -> TestResult { + let harness = start().await?; + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; + sender + .send(NodeMessage { + sequence: 2, + message_id: "node-message-2".to_owned(), + idempotency_key: "node-message-2".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node, + credential, + capabilities: Some(capabilities()), + capacity: Some(capacity()), + max_parallel: 1, + })), + }) + .await?; + assert!(matches!( + next(&mut inbound).await?.body, + Some(control_message::Body::Welcome(_)) + )); + + harness + .runs + .queue( + JobId::fresh(), + origin(), + serde_json::to_vec(&VersionedPlan::new(compile()))?, + ) + .await?; + let pushed = tokio::time::timeout(Duration::from_secs(1), next(&mut inbound)).await??; + assert!(matches!( + pushed.body, + Some(control_message::Body::Assignment(_)) + )); + Ok(()) +} + +/// A node that keeps beating is handed its next credential before the one it +/// holds runs out, and the new one is what opens the session after that. Without +/// this a node works for exactly one credential term and is then locked out of +/// the control plane for good, with no way back but a fresh enrolment token. +#[tokio::test] +async fn a_credential_near_expiry_is_replaced_over_the_open_session() -> TestResult { + let harness = start().await?; + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + + let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; + sender + .send(NodeMessage { + sequence: 2, + message_id: "node-message-2".to_owned(), + idempotency_key: "node-message-2".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node: node.clone(), + credential: credential.clone(), + capabilities: Some(capabilities()), + capacity: Some(capacity()), + max_parallel: 1, + })), + }) + .await?; + + let welcome = next(&mut inbound).await?; + assert!( + matches!(welcome.body, Some(control_message::Body::Welcome(_))), + "a fresh credential is not renewed, got {:?}", + welcome.body + ); + + // Age the credential by issuing it as though the term had nearly run out. + // Sixty seconds of life left is inside the renewal margin, so the next thing + // the node says has to be answered with a replacement. + let issued = SystemTime::now() - CREDENTIAL_TERM + Duration::from_secs(60); + let ageing = harness.nodes.rotate(node.parse()?, issued).await?; + sender + .send(NodeMessage { + sequence: 3, + message_id: "node-message-3".to_owned(), + idempotency_key: "node-message-3".to_owned(), + body: Some(node_message::Body::Heartbeat(Heartbeat { + held: Vec::new(), + capacity: Some(capacity()), + })), + }) + .await?; + + let answer = next(&mut inbound).await?; + let Some(control_message::Body::Rotated(rotated)) = answer.body else { + panic!("expected a replacement credential, got {:?}", answer.body); + }; + assert_ne!( + rotated.credential, + ageing.secret().expose(), + "a rotation that hands back the same credential renews nothing" + ); + assert_ne!(rotated.credential, credential); + let acknowledged = next(&mut inbound).await?; + assert!(matches!( + acknowledged.body, + Some(control_message::Body::Acknowledgement(_)) + )); + + sender + .send(NodeMessage { + sequence: 4, + message_id: "node-message-4".to_owned(), + idempotency_key: "node-message-4".to_owned(), + body: Some(node_message::Body::Heartbeat(Heartbeat { + held: Vec::new(), + capacity: Some(capacity()), + })), + }) + .await?; + let repeated = next(&mut inbound).await?; + let Some(control_message::Body::Rotated(repeated)) = repeated.body else { + panic!("expected the unconfirmed credential again"); + }; + assert_eq!(repeated.credential, rotated.credential); + + drop(sender); + drop(inbound); + wait_until_offline(&harness, node.parse()?).await?; + + // The point of the exercise: the credential the node was pushed is the one + // that gets it back in, and the one it replaced does not. + let (fresh_sender, fresh_receiver) = mpsc::channel(8); + let mut reopened = client + .open(ReceiverStream::new(fresh_receiver)) + .await? + .into_inner(); + fresh_sender + .send(NodeMessage { + sequence: 1, + message_id: "node-message-1".to_owned(), + idempotency_key: "node-message-1".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node: node.clone(), + credential: rotated.credential, + capabilities: Some(capabilities()), + capacity: Some(capacity()), + max_parallel: 1, + })), + }) + .await?; + assert!(matches!( + next(&mut reopened).await?.body, + Some(control_message::Body::Welcome(_)) + )); + + let (stale_sender, stale_receiver) = mpsc::channel(8); + let mut refused = client + .open(ReceiverStream::new(stale_receiver)) + .await? + .into_inner(); + stale_sender + .send(NodeMessage { + sequence: 1, + message_id: "node-message-1".to_owned(), + idempotency_key: "node-message-1".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node, + credential, + capabilities: Some(capabilities()), + capacity: Some(capacity()), + max_parallel: 1, + })), + }) + .await?; + let error = next(&mut refused) + .await + .expect_err("the credential that was replaced must stop working"); + assert!( + error.to_string().contains("not current"), + "unexpected error: {error}" + ); + + Ok(()) +} + +/// An identity handed out to something that never arrived has to settle, or the +/// sweep reports it as newly gone every fifteen seconds forever and the log +/// stops being worth reading. +#[tokio::test] +async fn a_node_that_enrolled_and_never_spoke_is_reported_gone_once() -> TestResult { + let harness = start().await?; + let token = harness.nodes.issue_token(Scope::Instance).await?; + let now = SystemTime::now(); + let (node, _) = harness.nodes.enrol(token.secret().expose(), now).await?; + + let deadline = Duration::from_secs(90); + let late = now + Duration::from_secs(600); + assert_eq!( + harness.nodes.sweep_silent(deadline, late).await?, + vec![node] + ); + assert!( + harness.nodes.sweep_silent(deadline, late).await?.is_empty(), + "a node already marked offline is not gone again" + ); + + Ok(()) +} + +fn progress(run: &str, job: &str, fence: u64, conclusion: Conclusion) -> NodeMessage { + let conclusion = match conclusion { + Conclusion::Success => syncode_control_node::wire::Conclusion::Success, + Conclusion::Failure => syncode_control_node::wire::Conclusion::Failure, + Conclusion::Cancelled => syncode_control_node::wire::Conclusion::Cancelled, + Conclusion::Skipped => syncode_control_node::wire::Conclusion::Skipped, + }; + NodeMessage { + sequence: 3, + message_id: "node-message-3".to_owned(), + idempotency_key: "node-message-3".to_owned(), + body: Some(node_message::Body::Progress(JobProgress { + run: run.to_owned(), + job: job.to_owned(), + conclusion: conclusion.into(), + fence, + outputs: Default::default(), + })), + } +} + +#[tokio::test] +async fn report_stating_a_grant_that_is_not_current_is_refused() -> TestResult { + let harness = start().await?; + let plan = VersionedPlan::new(compile()); + let run = harness + .runs + .queue(JobId::fresh(), origin(), serde_json::to_vec(&plan)?) + .await?; + + let holder = syncode_control_runs::NodeId::fresh(); + let assignment = harness + .runs + .take_next(holder) + .await? + .ok_or("nothing was queued")?; + + let error = harness + .runs + .finished( + run, + holder, + Fence::from(assignment.fence().get() + 1), + Conclusion::Success, + ) + .await + .expect_err("a grant nobody was given must not conclude a run"); + + assert!( + error.to_string().contains("not the current grant"), + "unexpected error: {error}" + ); + assert!(matches!( + harness.runs.state_of(run).await?, + RunState::Assigned(_) + )); + + Ok(()) +} + +#[tokio::test] +async fn a_session_without_a_credential_is_refused() -> TestResult { + let harness = start().await?; + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + + let (node, _) = enrolled(&harness, &sender, &mut inbound).await?; + sender + .send(NodeMessage { + sequence: 2, + message_id: "node-message-2".to_owned(), + idempotency_key: "node-message-2".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node, + credential: "not the one it was given".to_owned(), + capabilities: Some(capabilities()), + capacity: Some(capacity()), + max_parallel: 1, + })), + }) + .await?; + + let error = next(&mut inbound) + .await + .expect_err("a wrong credential must end the session"); + assert!( + error.to_string().contains("not current"), + "unexpected error: {error}" + ); + + Ok(()) +} + +#[tokio::test] +async fn a_node_that_does_not_say_how_much_room_it_has_is_refused() -> TestResult { + let harness = start().await?; + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + + let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; + sender + .send(NodeMessage { + sequence: 2, + message_id: "node-message-2".to_owned(), + idempotency_key: "node-message-2".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node, + credential, + capabilities: Some(capabilities()), + capacity: None, + max_parallel: 1, + })), + }) + .await?; + + let error = next(&mut inbound) + .await + .expect_err("a node without capacity must not open a session"); + assert!( + error.to_string().contains("how much room"), + "unexpected error: {error}" + ); + + Ok(()) +} + +#[tokio::test] +async fn a_failed_capacity_refresh_pauses_work_without_closing_the_session() -> TestResult { + let harness = start().await?; + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; + sender + .send(NodeMessage { + sequence: 2, + message_id: "node-message-2".to_owned(), + idempotency_key: "node-message-2".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node: node.clone(), + credential, + capabilities: Some(capabilities()), + capacity: Some(capacity()), + max_parallel: 1, + })), + }) + .await?; + assert!(matches!( + next(&mut inbound).await?.body, + Some(control_message::Body::Welcome(_)) + )); + + sender + .send(NodeMessage { + sequence: 3, + message_id: "node-message-3".to_owned(), + idempotency_key: "node-message-3".to_owned(), + body: Some(node_message::Body::Heartbeat(Heartbeat { + held: Vec::new(), + capacity: None, + })), + }) + .await?; + let node_id = node.parse()?; + for _ in 0..50 { + if !harness.nodes.employable(node_id).await? { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(!harness.nodes.employable(node_id).await?); + assert!(matches!( + next(&mut inbound).await?.body, + Some(control_message::Body::Acknowledgement(_)) + )); + harness + .runs + .queue( + JobId::fresh(), + origin(), + serde_json::to_vec(&VersionedPlan::new(compile()))?, + ) + .await?; + + assert!( + tokio::time::timeout(Duration::from_millis(100), next(&mut inbound)) + .await + .is_err(), + "an unmeasured node must stay connected without receiving work" + ); + + sender + .send(NodeMessage { + sequence: 4, + message_id: "node-message-4".to_owned(), + idempotency_key: "node-message-4".to_owned(), + body: Some(node_message::Body::Heartbeat(Heartbeat { + held: Vec::new(), + capacity: Some(capacity()), + })), + }) + .await?; + let answer = next(&mut inbound).await?; + assert!( + matches!(answer.body, Some(control_message::Body::Assignment(_))), + "a recovered capacity measurement must resume work" + ); + Ok(()) +} + +#[tokio::test] +async fn a_node_short_of_room_is_told_why_instead_of_being_given_work() -> TestResult { + let harness = start().await?; + let plan = VersionedPlan::new(compile()); + harness + .runs + .queue(JobId::fresh(), origin(), serde_json::to_vec(&plan)?) + .await?; + + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + + let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; + let cramped = Capacity { + build_volume_free_bytes: 1024, + ..capacity() + }; + sender + .send(NodeMessage { + sequence: 2, + message_id: "node-message-2".to_owned(), + idempotency_key: "node-message-2".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node, + credential, + capabilities: Some(capabilities()), + capacity: Some(cramped), + max_parallel: 1, + })), + }) + .await?; + + let welcome = next(&mut inbound).await?; + assert!(matches!( + welcome.body, + Some(control_message::Body::Welcome(_)) + )); + + let answer = next(&mut inbound).await?; + let Some(control_message::Body::Refused(refused)) = answer.body else { + panic!("expected a stated refusal, got {:?}", answer.body); + }; + assert!( + refused.reason.contains("below the"), + "unexpected reason: {}", + refused.reason + ); + + Ok(()) +} + +#[tokio::test] +async fn an_applied_message_is_deduplicated_after_reconnect() -> TestResult { + let harness = start().await?; + let (first_sender, first_receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut first = client + .open(ReceiverStream::new(first_receiver)) + .await? + .into_inner(); + let (node, credential) = enrolled(&harness, &first_sender, &mut first).await?; + first_sender + .send(NodeMessage { + sequence: 2, + message_id: "first-hello".to_owned(), + idempotency_key: "first-hello".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node: node.clone(), + credential: credential.clone(), + capabilities: Some(capabilities()), + capacity: Some(capacity()), + max_parallel: 1, + })), + }) + .await?; + let _ = next(&mut first).await?; + let repeated = NodeMessage { + sequence: 3, + message_id: "persistent-heartbeat".to_owned(), + idempotency_key: "persistent-heartbeat".to_owned(), + body: Some(node_message::Body::Heartbeat(Heartbeat { + held: Vec::new(), + capacity: None, + })), + }; + first_sender.send(repeated.clone()).await?; + let node_id = node.parse()?; + for _ in 0..50 { + if !harness.nodes.employable(node_id).await? { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(!harness.nodes.employable(node_id).await?); + + drop(first_sender); + drop(first); + wait_until_offline(&harness, node_id).await?; + + let (second_sender, second_receiver) = mpsc::channel(8); + let mut second_client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut second = second_client + .open(ReceiverStream::new(second_receiver)) + .await? + .into_inner(); + second_sender + .send(NodeMessage { + sequence: 1, + message_id: "second-hello".to_owned(), + idempotency_key: "second-hello".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node, + credential, + capabilities: Some(capabilities()), + capacity: Some(capacity()), + max_parallel: 1, + })), + }) + .await?; + let _ = next(&mut second).await?; + assert!(harness.nodes.employable(node_id).await?); + + second_sender + .send(NodeMessage { + sequence: 2, + ..repeated + }) + .await?; + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + harness.nodes.employable(node_id).await?, + "the duplicate must not clear capacity again" + ); + + second_sender + .send(NodeMessage { + sequence: 3, + message_id: "new-heartbeat".to_owned(), + idempotency_key: "new-heartbeat".to_owned(), + body: Some(node_message::Body::Heartbeat(Heartbeat { + held: Vec::new(), + capacity: None, + })), + }) + .await?; + for _ in 0..50 { + if !harness.nodes.employable(node_id).await? { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(!harness.nodes.employable(node_id).await?); + Ok(()) +} + +#[tokio::test] +async fn an_unacknowledged_assignment_is_retried_on_the_open_session() -> TestResult { + let harness = start().await?; + harness + .runs + .queue( + JobId::fresh(), + origin(), + serde_json::to_vec(&VersionedPlan::new(compile()))?, + ) + .await?; + + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; + sender + .send(NodeMessage { + sequence: 2, + message_id: "hello".to_owned(), + idempotency_key: "hello".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node: node.clone(), + credential, + capabilities: Some(capabilities()), + capacity: Some(capacity()), + max_parallel: 1, + })), + }) + .await?; + let _ = next(&mut inbound).await?; + let assignment = next(&mut inbound).await?; + assert!(matches!( + assignment.body, + Some(control_message::Body::Assignment(_)) + )); + + let retried = tokio::time::timeout(Duration::from_secs(3), next(&mut inbound)).await??; + assert_eq!(retried, assignment); + + sender + .send(NodeMessage { + sequence: 3, + message_id: "assignment-ack".to_owned(), + idempotency_key: "assignment-ack".to_owned(), + body: Some(node_message::Body::Acknowledgement(Acknowledgement { + message_id: assignment.message_id.clone(), + log_offset: 0, + acknowledged_outputs: Vec::new(), + conclusion: 0, + })), + }) + .await?; + let node = node.parse()?; + for _ in 0..50 { + if harness.nodes.outbound_pending(node).await?.is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(harness.nodes.outbound_pending(node).await?.is_empty()); + + let acknowledgement = Acknowledgement { + message_id: assignment.message_id, + log_offset: 0, + acknowledged_outputs: Vec::new(), + conclusion: 0, + }; + sender + .send(NodeMessage { + sequence: 4, + message_id: "retried-assignment-ack".to_owned(), + idempotency_key: "assignment-ack".to_owned(), + body: Some(node_message::Body::Acknowledgement(acknowledgement)), + }) + .await?; + sender + .send(NodeMessage { + sequence: 5, + message_id: "heartbeat-after-retried-ack".to_owned(), + idempotency_key: "heartbeat-after-retried-ack".to_owned(), + body: Some(node_message::Body::Heartbeat(Heartbeat { + held: Vec::new(), + capacity: Some(capacity()), + })), + }) + .await?; + assert!(matches!( + next(&mut inbound).await?.body, + Some(control_message::Body::Acknowledgement(_)) + )); + Ok(()) +} + +#[tokio::test] +async fn reconnect_does_not_restore_capacity_while_a_lease_is_held() -> TestResult { + let harness = start().await?; + let plan = serde_json::to_vec(&VersionedPlan::new(compile()))?; + let first_run = harness + .runs + .queue(JobId::fresh(), origin(), plan.clone()) + .await?; + let second_run = harness.runs.queue(JobId::fresh(), origin(), plan).await?; + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; + sender + .send(hello(2, node.clone(), credential.clone(), capacity())) + .await?; + let _ = next(&mut inbound).await?; + let assigned = next(&mut inbound).await?; + let Some(control_message::Body::Assignment(assignment)) = assigned.body.clone() else { + return Err("expected the first assignment".into()); + }; + assert_eq!(assignment.run, first_run.to_string()); + sender + .send(NodeMessage { + sequence: 3, + message_id: "assignment-ack".to_owned(), + idempotency_key: "assignment-ack".to_owned(), + body: Some(node_message::Body::Acknowledgement(Acknowledgement { + message_id: assigned.message_id, + log_offset: 0, + acknowledged_outputs: Vec::new(), + conclusion: 0, + })), + }) + .await?; + let node_id = node.parse()?; + for _ in 0..50 { + if harness.nodes.outbound_pending(node_id).await?.is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + drop(sender); + drop(inbound); + wait_until_offline(&harness, node_id).await?; + + let (sender, receiver) = mpsc::channel(8); + let mut reopened = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + sender.send(hello(1, node, credential, capacity())).await?; + let _ = next(&mut reopened).await?; + assert!( + tokio::time::timeout(Duration::from_millis(200), next(&mut reopened)) + .await + .is_err(), + "reconnect restored a slot still occupied by the first lease" + ); + + harness + .runs + .finished_job( + first_run, + assignment.job.parse()?, + node_id, + Fence::from(assignment.fence), + Conclusion::Success, + ) + .await?; + let next_assignment = + tokio::time::timeout(Duration::from_secs(1), next(&mut reopened)).await??; + let Some(control_message::Body::Assignment(assignment)) = next_assignment.body else { + return Err("expected the second assignment".into()); + }; + assert_eq!(assignment.run, second_run.to_string()); + Ok(()) +} + +#[tokio::test] +async fn a_silent_open_session_is_refused_after_its_lease_expires() -> TestResult { + let harness = start().await?; + let plan = serde_json::to_vec(&VersionedPlan::new(compile()))?; + let run = harness.runs.queue(JobId::fresh(), origin(), plan).await?; + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.address.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + let (node, credential) = enrolled(&harness, &sender, &mut inbound).await?; + sender + .send(hello(2, node.clone(), credential, capacity())) + .await?; + let _ = next(&mut inbound).await?; + let assignment = next(&mut inbound).await?; + assert!(matches!( + assignment.body, + Some(control_message::Body::Assignment(_)) + )); + + let node_id = node.parse()?; + let late = SystemTime::now() + Duration::from_secs(600); + assert_eq!( + harness + .nodes + .sweep_silent(Duration::from_secs(90), late) + .await?, + vec![node_id] + ); + assert_eq!(harness.nodes.lifecycle(node_id).await?, Lifecycle::Offline); + assert_eq!(harness.runs.reclaim_expired(late).await?, vec![run]); + + let refused = tokio::time::timeout(Duration::from_secs(1), next(&mut inbound)).await??; + assert!(matches!( + refused.body, + Some(control_message::Body::Refused(_)) + )); + assert_eq!(harness.runs.state_of(run).await?, RunState::Queued); + Ok(()) +} diff --git a/tests/webhook.rs b/tests/webhook.rs --- a/tests/webhook.rs +++ b/tests/webhook.rs @@ -1,680 +1,735 @@ -#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] - -#[path = "support/actions.rs"] -mod actions; -#[path = "support/repository.rs"] -mod repository; - -use std::error::Error; -use std::sync::Arc; -use std::time::Duration; - -use base64::Engine; -use base64::engine::general_purpose::STANDARD; -use hmac::{Hmac, KeyInit, Mac}; -use sha2::Sha256; -use syncode_control::repository::RepositoryContents; -use syncode_control::repository_grpc::NativeRepositoryContents; -use syncode_control::repository_sources::RepositorySources; -use syncode_control::webhook::{Intake, router}; -use syncode_control_node::wire::{ - Capabilities, Capacity, ControlMessage, Enrol, Hello, NodeMessage, control_message, - node_message, -}; -use syncode_control_node::{ - ArtifactTokenAuthority, CapabilityAuthority, GeneratedServer, NodeSessionClient, - NodeSessionServer, ProjectionClient, -}; -use syncode_control_nodes::{Ephemeral, Nodes, Scope}; -use syncode_control_runs::{Forgotten, NodeId, Runs}; -use syncode_workflow::VersionedPlan; -use syncode_workflow_github_actions::expression::ExpressionProgram; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::TcpListener; -use tokio::sync::mpsc; -use tokio_stream::StreamExt; -use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream}; -use tonic::Streaming; -use tonic::transport::Server; -use url::Url; - -use actions::FIXTURE_ACTIONS; - -type TestResult = Result>; - -const SECRET: &str = "the secret the hook was configured with"; -const COMMIT: &str = "9f2c1e4a7b3d5f6081a2c3d4e5f60718293a4b5c"; - -const WORKFLOW: &str = r#" -name: CI -on: - push: - branches: [main] - tags: ["v*"] - pull_request: - branches: [main] - paths: - - "src/**" - workflow_dispatch: - schedule: - - cron: "0 6 * * *" -jobs: - build: - runs-on: [self-hosted, linux] - steps: - - run: echo "the run came from a push" -"#; - -const OTHER_MANUAL_WORKFLOW: &str = r#" -name: Other manual workflow -on: - push: - workflow_dispatch: -jobs: - other: - runs-on: [self-hosted, linux] - steps: - - run: echo "the other workflow must not run" -"#; - -/// A forge that serves one workflow directory and has nothing in the other. -async fn forge() -> TestResult { - let listener = TcpListener::bind("127.0.0.1:0").await?; - let base = Url::parse(&format!("http://{}/", listener.local_addr()?))?; - let listing = format!( - r#"[{{"path":".gitea/workflows/ci.yml","type":"file","content":"{}"}},{{"path":".gitea/workflows/other.yml","type":"file","content":"{}"}}]"#, - STANDARD.encode(WORKFLOW), - STANDARD.encode(OTHER_MANUAL_WORKFLOW) - ); - - tokio::spawn(async move { - // Answers by what was asked for rather than in a fixed order: a push - // never asks for a file list, a pull request does. - let files = r#"[{"filename":"src/main.rs"}]"#.to_owned(); - loop { - let Ok((mut stream, _)) = listener.accept().await else { - return; - }; - let mut buffer = vec![0_u8; 4096]; - let read = stream.read(&mut buffer).await.unwrap_or(0); - let request = String::from_utf8_lossy(&buffer[..read]).to_string(); - let asked = request.lines().next().unwrap_or_default().to_owned(); - - let (status, body) = if asked.contains("/pulls/") { - (200, files.clone()) - } else if asked.contains(".gitea%2Fworkflows") || asked.contains(".gitea/workflows") { - (200, listing.clone()) - } else { - (404, String::from("{}")) - }; - - let response = format!( - "HTTP/1.1 {status} X\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", - body.len() - ); - let _ = stream.write_all(response.as_bytes()).await; - let _ = stream.shutdown().await; - } - }); - - Ok(base) -} - -struct Harness { - runs: Runs, - nodes: Nodes, - session: String, - events: String, -} - -/// Both ends of the control plane over one set of registries, exactly as the -/// binary wires them. -async fn start() -> TestResult { - start_mode(false).await -} - -async fn start_mode(shadow: bool) -> TestResult { - let runs = Runs::restored(Forgotten::default()).await?; - let nodes = Nodes::restored(Ephemeral::default()).await?; - let projection = projection_client().await?; - - let sessions = TcpListener::bind("127.0.0.1:0").await?; - let session = format!("http://{}", sessions.local_addr()?); - let served = (runs.clone(), nodes.clone()); - tokio::spawn(async move { - let session = if shadow { - NodeSessionServer::shadow( - served.0, - served.1, - capability_authority(), - artifact_authority(), - "http://127.0.0.1:1/".parse().expect("artifact URL"), - projection, - ) - } else { - NodeSessionServer::new( - served.0, - served.1, - capability_authority(), - artifact_authority(), - "http://127.0.0.1:1/".parse().expect("artifact URL"), - projection, - ) - }; - let _ = Server::builder() - .add_service(GeneratedServer::new(session)) - .serve_with_incoming(TcpListenerStream::new(sessions)) - .await; - }); - - let repository_listener = TcpListener::bind("127.0.0.1:0").await?; - let repository_endpoint = format!("http://{}", repository_listener.local_addr()?); - tokio::spawn(async move { - let _ = Server::builder() - .add_service(repository::service()) - .serve_with_incoming(TcpListenerStream::new(repository_listener)) - .await; - }); - let native = NativeRepositoryContents::connect(repository_endpoint).await?; - - let deliveries = TcpListener::bind("127.0.0.1:0").await?; - let events = format!("http://{}", deliveries.local_addr()?); - let intake = Arc::new(Intake::new( - RepositorySources::new( - native, - RepositoryContents::new(forge().await?, "a repository token".to_owned()), - ), - runs.clone(), - FIXTURE_ACTIONS, - SECRET.to_owned(), - )); - tokio::spawn(async move { - let _ = axum::serve(deliveries, router(intake)).await; - }); - - Ok(Harness { - runs, - nodes, - session, - events, - }) -} - -fn capability_authority() -> CapabilityAuthority { - CapabilityAuthority::new("test-capability-key").expect("capability key") -} - -fn artifact_authority() -> ArtifactTokenAuthority { - ArtifactTokenAuthority::new("test-artifact-key").expect("artifact key") -} - -async fn projection_client() -> TestResult { - let listener = TcpListener::bind("127.0.0.1:0").await?; - let base = Url::parse(&format!("http://{}/", listener.local_addr()?))?; - tokio::spawn(async move { - let app = axum::Router::new().fallback(|| async { axum::http::StatusCode::NO_CONTENT }); - if let Err(error) = axum::serve(listener, app).await { - eprintln!("projection fixture failed: {error}"); - } - }); - Ok(ProjectionClient::new(base, String::new())?) -} - -async fn open_node( - harness: &Harness, -) -> TestResult<(mpsc::Sender, Streaming)> { - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.session.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - let token = harness.nodes.issue_token(Scope::Instance).await?; - sender - .send(NodeMessage { - sequence: 1, - message_id: "node-message-1".to_owned(), - idempotency_key: "node-message-1".to_owned(), - body: Some(node_message::Body::Enrol(Enrol { - token: token.secret().expose().to_owned(), - })), - }) - .await?; - let message = next(&mut inbound).await?; - let Some(control_message::Body::Enrolled(enrolled)) = message.body else { - return Err("expected an enrolled node".into()); - }; - sender - .send(NodeMessage { - sequence: 2, - message_id: "node-message-2".to_owned(), - idempotency_key: "node-message-2".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node: enrolled.node, - credential: enrolled.credential, - capabilities: Some(Capabilities { - architecture: "arm64".to_owned(), - operating_system: "linux".to_owned(), - container_runtime: "docker".to_owned(), - container_runtime_version: "28.6.1".to_owned(), - cores: 2, - memory_bytes: 8 * 1024 * 1024 * 1024, - labels: vec!["self-hosted".to_owned(), "linux".to_owned()], - }), - capacity: Some(Capacity { - build_volume_free_bytes: 60 * 1024 * 1024 * 1024, - layer_store_bytes: 12 * 1024 * 1024 * 1024, - cache_volume_present: true, - cache_volume_total_bytes: 100, - cache_volume_used_bytes: 40, - cache_volume_path: "/var/cache/syncode".to_owned(), - cached_images: Vec::new(), - cached_actions: Vec::new(), - }), - max_parallel: 1, - })), - }) - .await?; - let welcome = next(&mut inbound).await?; - if !matches!(welcome.body, Some(control_message::Body::Welcome(_))) { - return Err("expected a welcome".into()); - } - Ok((sender, inbound)) -} - -fn push_body() -> String { - format!( - r#"{{"message_id":"018f47e2-b2c4-7f19-8a6d-13ef76c89211","protocol_version":"1.0","schema_version":1,"source_node_id":"018f47e2-b2c4-7f19-8a6d-13ef76c89212","repository_id":"{}","message_type":"repository.ref.updated","sequence":2,"term":1,"payload":{{"request":{{"principal_id":"018f47e2-b2c4-7f19-8a6d-13ef76c89213"}},"changes":[{{"name_hex":"726566732f68656164732f6d61696e","old":{{"kind":"object","object_id":"1111111111111111111111111111111111111111"}},"new":{{"kind":"object","object_id":"{COMMIT}"}}}}]}}}}"#, - repository::REPOSITORY - ) -} - -fn tag_body() -> String { - format!( - r#"{{"message_id":"018f47e2-b2c4-7f19-8a6d-13ef76c89211","protocol_version":"1.0","schema_version":1,"source_node_id":"018f47e2-b2c4-7f19-8a6d-13ef76c89212","repository_id":"{}","message_type":"repository.ref.updated","sequence":2,"term":1,"payload":{{"request":{{"principal_id":"018f47e2-b2c4-7f19-8a6d-13ef76c89213"}},"changes":[{{"name_hex":"726566732f746167732f76302e352e30","old":null,"new":{{"kind":"object","object_id":"{}"}}}}]}}}}"#, - repository::REPOSITORY, - repository::TAG_OBJECT - ) -} - -fn legacy_push_body() -> String { - format!( - r#"{{"ref":"refs/heads/main","after":"{COMMIT}","repository":{{"full_name":"syncode/demo"}},"commits":[{{"modified":["src/main.rs"]}}]}}"# - ) -} - -fn signature(body: &str) -> String { - let mut mac = Hmac::::new_from_slice(SECRET.as_bytes()).expect("key"); - mac.update(body.as_bytes()); - mac.finalize() - .into_bytes() - .iter() - .map(|byte| format!("{byte:02x}")) - .collect() -} - -async fn deliver(harness: &Harness, event: &str, body: &str, signed: bool) -> TestResult { - let signature = if signed { - signature(body) - } else { - signature("something else entirely") - }; - let response = reqwest::Client::new() - .post(format!("{}/events", harness.events)) - .header("x-syncode-event", event) - .header("x-syncode-signature", signature) - .header("x-syncode-delivery", format!("{event}-delivery")) - .header( - "x-syncode-workflow", - if event == "repository.ref.updated" { - ".gitea/workflows/ci.yml" - } else { - "" - }, - ) - .body(body.to_owned()) - .send() - .await?; - Ok(response.status().as_u16()) -} - -async fn deliver_native_pull_request(harness: &Harness, body: &str) -> TestResult { - let response = reqwest::Client::new() - .post(format!("{}/events", harness.events)) - .header("x-syncode-event", "pull_request") - .header("x-syncode-signature", signature(body)) - .header("x-syncode-delivery", "native-pull-request-delivery") - .header("x-syncode-repository-id", repository::REPOSITORY) - .header( - "x-syncode-before", - "1111111111111111111111111111111111111111", - ) - .header("x-syncode-after", COMMIT) - .body(body.to_owned()) - .send() - .await?; - Ok(response.status().as_u16()) -} - -async fn next(stream: &mut Streaming) -> TestResult { - Ok(stream - .next() - .await - .ok_or("the control plane closed the stream")??) -} - -#[tokio::test] -async fn a_push_becomes_a_plan_the_node_is_handed() -> TestResult { - let harness = start().await?; - - let body = push_body(); - assert_eq!( - deliver(&harness, "repository.ref.updated", &body, true).await?, - 202 - ); - - let (sender, receiver) = mpsc::channel(8); - let mut client = NodeSessionClient::connect(harness.session.clone()).await?; - let mut inbound = client - .open(ReceiverStream::new(receiver)) - .await? - .into_inner(); - - let token = harness.nodes.issue_token(Scope::Instance).await?; - sender - .send(NodeMessage { - sequence: 1, - message_id: "node-message-1".to_owned(), - idempotency_key: "node-message-1".to_owned(), - body: Some(node_message::Body::Enrol(Enrol { - token: token.secret().expose().to_owned(), - })), - }) - .await?; - let message = next(&mut inbound).await?; - let Some(control_message::Body::Enrolled(enrolled)) = message.body else { - panic!("expected an identity, got {:?}", message.body); - }; - - sender - .send(NodeMessage { - sequence: 2, - message_id: "node-message-2".to_owned(), - idempotency_key: "node-message-2".to_owned(), - body: Some(node_message::Body::Hello(Hello { - node: enrolled.node, - credential: enrolled.credential, - capabilities: Some(Capabilities { - architecture: "arm64".to_owned(), - operating_system: "linux".to_owned(), - container_runtime: "docker".to_owned(), - container_runtime_version: "28.6.1".to_owned(), - cores: 2, - memory_bytes: 8 * 1024 * 1024 * 1024, - labels: vec!["self-hosted".to_owned(), "linux".to_owned()], - }), - capacity: Some(Capacity { - build_volume_free_bytes: 60 * 1024 * 1024 * 1024, - layer_store_bytes: 12 * 1024 * 1024 * 1024, - cache_volume_present: true, - cache_volume_total_bytes: 100, - cache_volume_used_bytes: 40, - cache_volume_path: "/var/cache/syncode".to_owned(), - cached_images: Vec::new(), - cached_actions: Vec::new(), - }), - max_parallel: 1, - })), - }) - .await?; - - let welcome = next(&mut inbound).await?; - assert!(matches!( - welcome.body, - Some(control_message::Body::Welcome(_)) - )); - - let assignment = next(&mut inbound).await?; - let Some(control_message::Body::Assignment(assignment)) = assignment.body else { - panic!("expected the plan the push produced, got {assignment:?}"); - }; - // The node is handed a plan, not the workflow file the push carried. - let plan: VersionedPlan = serde_json::from_slice(&assignment.plan)?; - assert_eq!(plan.schema(), syncode_workflow::PlanSchemaVersion::CURRENT); - - // A plan says nothing about what it is being built from, so the assignment - // states it: without this a job cannot check anything out. - let origin = assignment.origin.ok_or("the assignment stated no origin")?; - assert_eq!(origin.number, 1); - assert_eq!(origin.repository, repository::REPOSITORY); - assert_eq!(origin.commit, COMMIT); - assert_eq!(origin.reference, "refs/heads/main"); - assert_eq!(origin.event, "push"); - - Ok(()) -} - -#[tokio::test] -async fn an_annotated_tag_push_uses_the_target_commit_and_tag_reference() -> TestResult { - let harness = start().await?; - let body = tag_body(); - - assert_eq!( - deliver(&harness, "repository.ref.updated", &body, true).await?, - 202 - ); - let assignment = harness - .runs - .take_next(NodeId::fresh()) - .await? - .ok_or("the tag push produced no run")?; - - assert_eq!(assignment.origin().commit(), COMMIT); - assert_eq!(assignment.origin().reference(), "refs/tags/v0.5.0"); - assert_eq!(assignment.origin().event(), "push"); - Ok(()) -} - -#[tokio::test] -async fn a_delivery_nobody_signed_queues_nothing() -> TestResult { - let harness = start().await?; - - let body = push_body(); - assert_eq!( - deliver(&harness, "repository.ref.updated", &body, false).await?, - 401 - ); - assert!( - harness - .runs - .take_next(syncode_control_runs::NodeId::fresh()) - .await? - .is_none() - ); - - Ok(()) -} - -#[tokio::test] -async fn a_retried_delivery_does_not_duplicate_its_run() -> TestResult { - let harness = start().await?; - let body = push_body(); - assert_eq!( - deliver(&harness, "repository.ref.updated", &body, true).await?, - 202 - ); - assert_eq!( - deliver(&harness, "repository.ref.updated", &body, true).await?, - 202 - ); - - let first = harness - .runs - .take_next(NodeId::fresh()) - .await? - .ok_or("the delivery produced no run")?; - assert!(harness.runs.take_next(NodeId::fresh()).await?.is_none()); - assert_eq!(first.origin().event(), "push"); - Ok(()) -} - -#[tokio::test] -async fn an_event_the_workflow_does_not_declare_is_accepted_and_ignored() -> TestResult { - let harness = start().await?; - - let body = push_body(); - assert_eq!(deliver(&harness, "issues", &body, true).await?, 202); - assert!( - harness - .runs - .take_next(syncode_control_runs::NodeId::fresh()) - .await? - .is_none() - ); - - Ok(()) -} - -#[tokio::test] -async fn a_legacy_forge_push_is_accepted_without_a_duplicate_run() -> TestResult { - let harness = start().await?; - let body = legacy_push_body(); - assert_eq!(deliver(&harness, "push", &body, true).await?, 202); - assert!(harness.runs.take_next(NodeId::fresh()).await?.is_none()); - Ok(()) -} - -fn pull_request_body(action: &str) -> String { - format!( - r#"{{"action":"{action}","number":7,"repository":{{"full_name":"syncode/demo"}},"pull_request":{{"head":{{"sha":"{COMMIT}"}},"base":{{"ref":"main"}}}}}}"# - ) -} - -#[tokio::test] -async fn a_pull_request_becomes_a_run_using_the_files_it_touches() -> TestResult { - let harness = start().await?; - - let body = pull_request_body("opened"); - assert_eq!(deliver(&harness, "pull_request", &body, true).await?, 202); - - // The workflow filters on `paths: src/**`. The webhook body carries no file - // list, so unless the control plane asks the forge for one, this run does - // not exist and nothing says why. - let assignment = harness - .runs - .take_next(syncode_control_runs::NodeId::fresh()) - .await? - .ok_or("the pull request produced no run")?; - let plan: VersionedPlan = serde_json::from_slice(assignment.plan())?; - assert_eq!(plan.schema(), syncode_workflow::PlanSchemaVersion::CURRENT); - - // A pull request head sits on no branch this control plane knows, but the - // forge publishes it under the pull request, which is what a checkout can - // fetch. - assert_eq!(assignment.origin().reference(), "refs/pull/7/head"); - assert_eq!(assignment.origin().event(), "pull_request"); - Ok(()) -} - -#[tokio::test] -async fn a_repository_plane_pull_request_reads_content_over_grpc() -> TestResult { - let harness = start().await?; - - let body = pull_request_body("opened"); - assert_eq!(deliver_native_pull_request(&harness, &body).await?, 202); - - let assignment = harness - .runs - .take_next(syncode_control_runs::NodeId::fresh()) - .await? - .ok_or("the native pull request produced no run")?; - assert_eq!(assignment.origin().repository(), "syncode/demo"); - assert_eq!(assignment.origin().reference(), "refs/pull/7/head"); - Ok(()) -} - -#[tokio::test] -async fn closing_a_pull_request_starts_nothing() -> TestResult { - let harness = start().await?; - - let body = pull_request_body("closed"); - assert_eq!(deliver(&harness, "pull_request", &body, true).await?, 202); - assert!( - harness - .runs - .take_next(syncode_control_runs::NodeId::fresh()) - .await? - .is_none() - ); - Ok(()) -} - -#[tokio::test] -async fn manual_and_scheduled_deliveries_become_runs() -> TestResult { - for event in ["workflow_dispatch", "schedule"] { - let harness = start().await?; - let body = format!( - r#"{{"ref":"refs/heads/main","after":"{COMMIT}","workflow":".gitea/workflows/ci.yml","repository":{{"full_name":"syncode/demo"}}}}"# - ); - assert_eq!(deliver(&harness, event, &body, true).await?, 202); - let assignment = harness - .runs - .take_next(syncode_control_runs::NodeId::fresh()) - .await? - .ok_or("event produced no run")?; - assert_eq!(assignment.origin().event(), event); - assert_eq!(assignment.origin().reference(), "refs/heads/main"); - assert!( - harness - .runs - .take_next(syncode_control_runs::NodeId::fresh()) - .await? - .is_none(), - "a requested event started a workflow other than the selected file" - ); - } - Ok(()) -} - -#[tokio::test] -async fn a_manual_delivery_can_select_a_tag() -> TestResult { - let harness = start().await?; - let body = format!( - r#"{{"ref":"refs/tags/v0.4.0","after":"{COMMIT}","workflow":".gitea/workflows/ci.yml","repository":{{"full_name":"syncode/demo"}}}}"# - ); - - assert_eq!( - deliver(&harness, "workflow_dispatch", &body, true).await?, - 202 - ); - let assignment = harness - .runs - .take_next(NodeId::fresh()) - .await? - .ok_or("manual tag event produced no run")?; - assert_eq!(assignment.origin().reference(), "refs/tags/v0.4.0"); - Ok(()) -} - -#[tokio::test] -async fn shadow_mode_persists_the_run_but_never_assigns_it() -> TestResult { - let harness = start_mode(true).await?; - let body = push_body(); - assert_eq!( - deliver(&harness, "repository.ref.updated", &body, true).await?, - 202 - ); - let (_sender, mut inbound) = open_node(&harness).await?; - - assert!( - tokio::time::timeout(Duration::from_millis(100), inbound.next()) - .await - .is_err(), - "shadow mode must not assign the run" - ); - assert!( - harness.runs.take_next(NodeId::fresh()).await?.is_some(), - "the shadow run must still exist in the aggregate" - ); - Ok(()) -} +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] + +#[path = "support/actions.rs"] +mod actions; +#[path = "support/repository.rs"] +mod repository; + +use std::error::Error; +use std::sync::Arc; +use std::time::Duration; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use hmac::{Hmac, KeyInit, Mac}; +use sha2::Sha256; +use syncode_control::repository::RepositoryContents; +use syncode_control::repository_grpc::NativeRepositoryContents; +use syncode_control::repository_sources::RepositorySources; +use syncode_control::webhook::{Intake, router}; +use syncode_control_node::identity_wire::identity_server::{Identity, IdentityServer}; +use syncode_control_node::identity_wire::{ + CheckCapabilityRequest, CheckCapabilityResponse, GetRepositoryCoordinatesRequest, + GetRepositoryCoordinatesResponse, ResolveRepositoryRequest, ResolveRepositoryResponse, + ValidateSessionRequest, ValidateSessionResponse, +}; +use syncode_control_node::wire::{ + Capabilities, Capacity, ControlMessage, Enrol, Hello, NodeMessage, control_message, + node_message, +}; +use syncode_control_node::{ + ArtifactTokenAuthority, CapabilityAuthority, GeneratedServer, NodeSessionClient, + NodeSessionServer, ProjectionClient, +}; +use syncode_control_nodes::{Ephemeral, Nodes, Scope}; +use syncode_control_runs::{Forgotten, NodeId, Runs}; +use syncode_workflow::VersionedPlan; +use syncode_workflow_github_actions::expression::ExpressionProgram; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; +use tokio::sync::mpsc; +use tokio_stream::StreamExt; +use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream}; +use tonic::transport::Server; +use tonic::{Request, Response, Status, Streaming}; +use url::Url; + +use actions::FIXTURE_ACTIONS; + +type TestResult = Result>; + +struct ProjectionIdentity; + +#[tonic::async_trait] +impl Identity for ProjectionIdentity { + async fn validate_session( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("validate_session")) + } + + async fn check_capability( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("check_capability")) + } + + async fn resolve_repository( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("resolve_repository")) + } + + async fn get_repository_coordinates( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(GetRepositoryCoordinatesResponse { + owner: "syncode".to_owned(), + name: "fixture".to_owned(), + })) + } +} + +const SECRET: &str = "the secret the hook was configured with"; +const COMMIT: &str = "9f2c1e4a7b3d5f6081a2c3d4e5f60718293a4b5c"; + +const WORKFLOW: &str = r#" +name: CI +on: + push: + branches: [main] + tags: ["v*"] + pull_request: + branches: [main] + paths: + - "src/**" + workflow_dispatch: + schedule: + - cron: "0 6 * * *" +jobs: + build: + runs-on: [self-hosted, linux] + steps: + - run: echo "the run came from a push" +"#; + +const OTHER_MANUAL_WORKFLOW: &str = r#" +name: Other manual workflow +on: + push: + workflow_dispatch: +jobs: + other: + runs-on: [self-hosted, linux] + steps: + - run: echo "the other workflow must not run" +"#; + +/// A forge that serves one workflow directory and has nothing in the other. +async fn forge() -> TestResult { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let base = Url::parse(&format!("http://{}/", listener.local_addr()?))?; + let listing = format!( + r#"[{{"path":".gitea/workflows/ci.yml","type":"file","content":"{}"}},{{"path":".gitea/workflows/other.yml","type":"file","content":"{}"}}]"#, + STANDARD.encode(WORKFLOW), + STANDARD.encode(OTHER_MANUAL_WORKFLOW) + ); + + tokio::spawn(async move { + // Answers by what was asked for rather than in a fixed order: a push + // never asks for a file list, a pull request does. + let files = r#"[{"filename":"src/main.rs"}]"#.to_owned(); + loop { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + let mut buffer = vec![0_u8; 4096]; + let read = stream.read(&mut buffer).await.unwrap_or(0); + let request = String::from_utf8_lossy(&buffer[..read]).to_string(); + let asked = request.lines().next().unwrap_or_default().to_owned(); + + let (status, body) = if asked.contains("/pulls/") { + (200, files.clone()) + } else if asked.contains(".gitea%2Fworkflows") || asked.contains(".gitea/workflows") { + (200, listing.clone()) + } else { + (404, String::from("{}")) + }; + + let response = format!( + "HTTP/1.1 {status} X\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", + body.len() + ); + let _ = stream.write_all(response.as_bytes()).await; + let _ = stream.shutdown().await; + } + }); + + Ok(base) +} + +struct Harness { + runs: Runs, + nodes: Nodes, + session: String, + events: String, +} + +/// Both ends of the control plane over one set of registries, exactly as the +/// binary wires them. +async fn start() -> TestResult { + start_mode(false).await +} + +async fn start_mode(shadow: bool) -> TestResult { + let runs = Runs::restored(Forgotten::default()).await?; + let nodes = Nodes::restored(Ephemeral::default()).await?; + let projection = projection_client().await?; + + let sessions = TcpListener::bind("127.0.0.1:0").await?; + let session = format!("http://{}", sessions.local_addr()?); + let served = (runs.clone(), nodes.clone()); + tokio::spawn(async move { + let session = if shadow { + NodeSessionServer::shadow( + served.0, + served.1, + capability_authority(), + artifact_authority(), + "http://127.0.0.1:1/".parse().expect("artifact URL"), + projection, + ) + } else { + NodeSessionServer::new( + served.0, + served.1, + capability_authority(), + artifact_authority(), + "http://127.0.0.1:1/".parse().expect("artifact URL"), + projection, + ) + }; + let _ = Server::builder() + .add_service(GeneratedServer::new(session)) + .serve_with_incoming(TcpListenerStream::new(sessions)) + .await; + }); + + let repository_listener = TcpListener::bind("127.0.0.1:0").await?; + let repository_endpoint = format!("http://{}", repository_listener.local_addr()?); + tokio::spawn(async move { + let _ = Server::builder() + .add_service(repository::service()) + .serve_with_incoming(TcpListenerStream::new(repository_listener)) + .await; + }); + let native = NativeRepositoryContents::connect(repository_endpoint).await?; + + let deliveries = TcpListener::bind("127.0.0.1:0").await?; + let events = format!("http://{}", deliveries.local_addr()?); + let intake = Arc::new(Intake::new( + RepositorySources::new( + native, + RepositoryContents::new(forge().await?, "a repository token".to_owned()), + ), + runs.clone(), + FIXTURE_ACTIONS, + SECRET.to_owned(), + )); + tokio::spawn(async move { + let _ = axum::serve(deliveries, router(intake)).await; + }); + + Ok(Harness { + runs, + nodes, + session, + events, + }) +} + +fn capability_authority() -> CapabilityAuthority { + CapabilityAuthority::new("test-capability-key").expect("capability key") +} + +fn artifact_authority() -> ArtifactTokenAuthority { + ArtifactTokenAuthority::new("test-artifact-key").expect("artifact key") +} + +async fn projection_client() -> TestResult { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let base = Url::parse(&format!("http://{}/", listener.local_addr()?))?; + tokio::spawn(async move { + let app = axum::Router::new().fallback(|| async { axum::http::StatusCode::NO_CONTENT }); + if let Err(error) = axum::serve(listener, app).await { + eprintln!("projection fixture failed: {error}"); + } + }); + let identity_listener = TcpListener::bind("127.0.0.1:0").await?; + let identity_endpoint = format!("http://{}", identity_listener.local_addr()?); + tokio::spawn(async move { + let _ = Server::builder() + .add_service(IdentityServer::new(ProjectionIdentity)) + .serve_with_incoming(TcpListenerStream::new(identity_listener)) + .await; + }); + Ok(ProjectionClient::new( + base, + String::new(), + identity_endpoint, + String::new(), + )?) +} + +async fn open_node( + harness: &Harness, +) -> TestResult<(mpsc::Sender, Streaming)> { + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.session.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + let token = harness.nodes.issue_token(Scope::Instance).await?; + sender + .send(NodeMessage { + sequence: 1, + message_id: "node-message-1".to_owned(), + idempotency_key: "node-message-1".to_owned(), + body: Some(node_message::Body::Enrol(Enrol { + token: token.secret().expose().to_owned(), + })), + }) + .await?; + let message = next(&mut inbound).await?; + let Some(control_message::Body::Enrolled(enrolled)) = message.body else { + return Err("expected an enrolled node".into()); + }; + sender + .send(NodeMessage { + sequence: 2, + message_id: "node-message-2".to_owned(), + idempotency_key: "node-message-2".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node: enrolled.node, + credential: enrolled.credential, + capabilities: Some(Capabilities { + architecture: "arm64".to_owned(), + operating_system: "linux".to_owned(), + container_runtime: "docker".to_owned(), + container_runtime_version: "28.6.1".to_owned(), + cores: 2, + memory_bytes: 8 * 1024 * 1024 * 1024, + labels: vec!["self-hosted".to_owned(), "linux".to_owned()], + }), + capacity: Some(Capacity { + build_volume_free_bytes: 60 * 1024 * 1024 * 1024, + layer_store_bytes: 12 * 1024 * 1024 * 1024, + cache_volume_present: true, + cache_volume_total_bytes: 100, + cache_volume_used_bytes: 40, + cache_volume_path: "/var/cache/syncode".to_owned(), + cached_images: Vec::new(), + cached_actions: Vec::new(), + }), + max_parallel: 1, + })), + }) + .await?; + let welcome = next(&mut inbound).await?; + if !matches!(welcome.body, Some(control_message::Body::Welcome(_))) { + return Err("expected a welcome".into()); + } + Ok((sender, inbound)) +} + +fn push_body() -> String { + format!( + r#"{{"message_id":"018f47e2-b2c4-7f19-8a6d-13ef76c89211","protocol_version":"1.0","schema_version":1,"source_node_id":"018f47e2-b2c4-7f19-8a6d-13ef76c89212","repository_id":"{}","message_type":"repository.ref.updated","sequence":2,"term":1,"payload":{{"request":{{"principal_id":"018f47e2-b2c4-7f19-8a6d-13ef76c89213"}},"changes":[{{"name_hex":"726566732f68656164732f6d61696e","old":{{"kind":"object","object_id":"1111111111111111111111111111111111111111"}},"new":{{"kind":"object","object_id":"{COMMIT}"}}}}]}}}}"#, + repository::REPOSITORY + ) +} + +fn tag_body() -> String { + format!( + r#"{{"message_id":"018f47e2-b2c4-7f19-8a6d-13ef76c89211","protocol_version":"1.0","schema_version":1,"source_node_id":"018f47e2-b2c4-7f19-8a6d-13ef76c89212","repository_id":"{}","message_type":"repository.ref.updated","sequence":2,"term":1,"payload":{{"request":{{"principal_id":"018f47e2-b2c4-7f19-8a6d-13ef76c89213"}},"changes":[{{"name_hex":"726566732f746167732f76302e352e30","old":null,"new":{{"kind":"object","object_id":"{}"}}}}]}}}}"#, + repository::REPOSITORY, + repository::TAG_OBJECT + ) +} + +fn legacy_push_body() -> String { + format!( + r#"{{"ref":"refs/heads/main","after":"{COMMIT}","repository":{{"full_name":"syncode/demo"}},"commits":[{{"modified":["src/main.rs"]}}]}}"# + ) +} + +fn signature(body: &str) -> String { + let mut mac = Hmac::::new_from_slice(SECRET.as_bytes()).expect("key"); + mac.update(body.as_bytes()); + mac.finalize() + .into_bytes() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +async fn deliver(harness: &Harness, event: &str, body: &str, signed: bool) -> TestResult { + let signature = if signed { + signature(body) + } else { + signature("something else entirely") + }; + let response = reqwest::Client::new() + .post(format!("{}/events", harness.events)) + .header("x-syncode-event", event) + .header("x-syncode-signature", signature) + .header("x-syncode-delivery", format!("{event}-delivery")) + .header( + "x-syncode-workflow", + if event == "repository.ref.updated" { + ".gitea/workflows/ci.yml" + } else { + "" + }, + ) + .body(body.to_owned()) + .send() + .await?; + Ok(response.status().as_u16()) +} + +async fn deliver_native_pull_request(harness: &Harness, body: &str) -> TestResult { + let response = reqwest::Client::new() + .post(format!("{}/events", harness.events)) + .header("x-syncode-event", "pull_request") + .header("x-syncode-signature", signature(body)) + .header("x-syncode-delivery", "native-pull-request-delivery") + .header("x-syncode-repository-id", repository::REPOSITORY) + .header( + "x-syncode-before", + "1111111111111111111111111111111111111111", + ) + .header("x-syncode-after", COMMIT) + .body(body.to_owned()) + .send() + .await?; + Ok(response.status().as_u16()) +} + +async fn next(stream: &mut Streaming) -> TestResult { + Ok(stream + .next() + .await + .ok_or("the control plane closed the stream")??) +} + +#[tokio::test] +async fn a_push_becomes_a_plan_the_node_is_handed() -> TestResult { + let harness = start().await?; + + let body = push_body(); + assert_eq!( + deliver(&harness, "repository.ref.updated", &body, true).await?, + 202 + ); + + let (sender, receiver) = mpsc::channel(8); + let mut client = NodeSessionClient::connect(harness.session.clone()).await?; + let mut inbound = client + .open(ReceiverStream::new(receiver)) + .await? + .into_inner(); + + let token = harness.nodes.issue_token(Scope::Instance).await?; + sender + .send(NodeMessage { + sequence: 1, + message_id: "node-message-1".to_owned(), + idempotency_key: "node-message-1".to_owned(), + body: Some(node_message::Body::Enrol(Enrol { + token: token.secret().expose().to_owned(), + })), + }) + .await?; + let message = next(&mut inbound).await?; + let Some(control_message::Body::Enrolled(enrolled)) = message.body else { + panic!("expected an identity, got {:?}", message.body); + }; + + sender + .send(NodeMessage { + sequence: 2, + message_id: "node-message-2".to_owned(), + idempotency_key: "node-message-2".to_owned(), + body: Some(node_message::Body::Hello(Hello { + node: enrolled.node, + credential: enrolled.credential, + capabilities: Some(Capabilities { + architecture: "arm64".to_owned(), + operating_system: "linux".to_owned(), + container_runtime: "docker".to_owned(), + container_runtime_version: "28.6.1".to_owned(), + cores: 2, + memory_bytes: 8 * 1024 * 1024 * 1024, + labels: vec!["self-hosted".to_owned(), "linux".to_owned()], + }), + capacity: Some(Capacity { + build_volume_free_bytes: 60 * 1024 * 1024 * 1024, + layer_store_bytes: 12 * 1024 * 1024 * 1024, + cache_volume_present: true, + cache_volume_total_bytes: 100, + cache_volume_used_bytes: 40, + cache_volume_path: "/var/cache/syncode".to_owned(), + cached_images: Vec::new(), + cached_actions: Vec::new(), + }), + max_parallel: 1, + })), + }) + .await?; + + let welcome = next(&mut inbound).await?; + assert!(matches!( + welcome.body, + Some(control_message::Body::Welcome(_)) + )); + + let assignment = next(&mut inbound).await?; + let Some(control_message::Body::Assignment(assignment)) = assignment.body else { + panic!("expected the plan the push produced, got {assignment:?}"); + }; + // The node is handed a plan, not the workflow file the push carried. + let plan: VersionedPlan = serde_json::from_slice(&assignment.plan)?; + assert_eq!(plan.schema(), syncode_workflow::PlanSchemaVersion::CURRENT); + + // A plan says nothing about what it is being built from, so the assignment + // states it: without this a job cannot check anything out. + let origin = assignment.origin.ok_or("the assignment stated no origin")?; + assert_eq!(origin.number, 1); + assert_eq!(origin.repository, repository::REPOSITORY); + assert_eq!(origin.commit, COMMIT); + assert_eq!(origin.reference, "refs/heads/main"); + assert_eq!(origin.event, "push"); + + Ok(()) +} + +#[tokio::test] +async fn an_annotated_tag_push_uses_the_target_commit_and_tag_reference() -> TestResult { + let harness = start().await?; + let body = tag_body(); + + assert_eq!( + deliver(&harness, "repository.ref.updated", &body, true).await?, + 202 + ); + let assignment = harness + .runs + .take_next(NodeId::fresh()) + .await? + .ok_or("the tag push produced no run")?; + + assert_eq!(assignment.origin().commit(), COMMIT); + assert_eq!(assignment.origin().reference(), "refs/tags/v0.5.0"); + assert_eq!(assignment.origin().event(), "push"); + Ok(()) +} + +#[tokio::test] +async fn a_delivery_nobody_signed_queues_nothing() -> TestResult { + let harness = start().await?; + + let body = push_body(); + assert_eq!( + deliver(&harness, "repository.ref.updated", &body, false).await?, + 401 + ); + assert!( + harness + .runs + .take_next(syncode_control_runs::NodeId::fresh()) + .await? + .is_none() + ); + + Ok(()) +} + +#[tokio::test] +async fn a_retried_delivery_does_not_duplicate_its_run() -> TestResult { + let harness = start().await?; + let body = push_body(); + assert_eq!( + deliver(&harness, "repository.ref.updated", &body, true).await?, + 202 + ); + assert_eq!( + deliver(&harness, "repository.ref.updated", &body, true).await?, + 202 + ); + + let first = harness + .runs + .take_next(NodeId::fresh()) + .await? + .ok_or("the delivery produced no run")?; + assert!(harness.runs.take_next(NodeId::fresh()).await?.is_none()); + assert_eq!(first.origin().event(), "push"); + Ok(()) +} + +#[tokio::test] +async fn an_event_the_workflow_does_not_declare_is_accepted_and_ignored() -> TestResult { + let harness = start().await?; + + let body = push_body(); + assert_eq!(deliver(&harness, "issues", &body, true).await?, 202); + assert!( + harness + .runs + .take_next(syncode_control_runs::NodeId::fresh()) + .await? + .is_none() + ); + + Ok(()) +} + +#[tokio::test] +async fn a_legacy_forge_push_is_accepted_without_a_duplicate_run() -> TestResult { + let harness = start().await?; + let body = legacy_push_body(); + assert_eq!(deliver(&harness, "push", &body, true).await?, 202); + assert!(harness.runs.take_next(NodeId::fresh()).await?.is_none()); + Ok(()) +} + +fn pull_request_body(action: &str) -> String { + format!( + r#"{{"action":"{action}","number":7,"repository":{{"full_name":"syncode/demo"}},"pull_request":{{"head":{{"sha":"{COMMIT}"}},"base":{{"ref":"main"}}}}}}"# + ) +} + +#[tokio::test] +async fn a_pull_request_becomes_a_run_using_the_files_it_touches() -> TestResult { + let harness = start().await?; + + let body = pull_request_body("opened"); + assert_eq!(deliver(&harness, "pull_request", &body, true).await?, 202); + + // The workflow filters on `paths: src/**`. The webhook body carries no file + // list, so unless the control plane asks the forge for one, this run does + // not exist and nothing says why. + let assignment = harness + .runs + .take_next(syncode_control_runs::NodeId::fresh()) + .await? + .ok_or("the pull request produced no run")?; + let plan: VersionedPlan = serde_json::from_slice(assignment.plan())?; + assert_eq!(plan.schema(), syncode_workflow::PlanSchemaVersion::CURRENT); + + // A pull request head sits on no branch this control plane knows, but the + // forge publishes it under the pull request, which is what a checkout can + // fetch. + assert_eq!(assignment.origin().reference(), "refs/pull/7/head"); + assert_eq!(assignment.origin().event(), "pull_request"); + Ok(()) +} + +#[tokio::test] +async fn a_repository_plane_pull_request_reads_content_over_grpc() -> TestResult { + let harness = start().await?; + + let body = pull_request_body("opened"); + assert_eq!(deliver_native_pull_request(&harness, &body).await?, 202); + + let assignment = harness + .runs + .take_next(syncode_control_runs::NodeId::fresh()) + .await? + .ok_or("the native pull request produced no run")?; + assert_eq!(assignment.origin().repository(), "syncode/demo"); + assert_eq!(assignment.origin().reference(), "refs/pull/7/head"); + Ok(()) +} + +#[tokio::test] +async fn closing_a_pull_request_starts_nothing() -> TestResult { + let harness = start().await?; + + let body = pull_request_body("closed"); + assert_eq!(deliver(&harness, "pull_request", &body, true).await?, 202); + assert!( + harness + .runs + .take_next(syncode_control_runs::NodeId::fresh()) + .await? + .is_none() + ); + Ok(()) +} + +#[tokio::test] +async fn manual_and_scheduled_deliveries_become_runs() -> TestResult { + for event in ["workflow_dispatch", "schedule"] { + let harness = start().await?; + let body = format!( + r#"{{"ref":"refs/heads/main","after":"{COMMIT}","workflow":".gitea/workflows/ci.yml","repository":{{"full_name":"syncode/demo"}}}}"# + ); + assert_eq!(deliver(&harness, event, &body, true).await?, 202); + let assignment = harness + .runs + .take_next(syncode_control_runs::NodeId::fresh()) + .await? + .ok_or("event produced no run")?; + assert_eq!(assignment.origin().event(), event); + assert_eq!(assignment.origin().reference(), "refs/heads/main"); + assert!( + harness + .runs + .take_next(syncode_control_runs::NodeId::fresh()) + .await? + .is_none(), + "a requested event started a workflow other than the selected file" + ); + } + Ok(()) +} + +#[tokio::test] +async fn a_manual_delivery_can_select_a_tag() -> TestResult { + let harness = start().await?; + let body = format!( + r#"{{"ref":"refs/tags/v0.4.0","after":"{COMMIT}","workflow":".gitea/workflows/ci.yml","repository":{{"full_name":"syncode/demo"}}}}"# + ); + + assert_eq!( + deliver(&harness, "workflow_dispatch", &body, true).await?, + 202 + ); + let assignment = harness + .runs + .take_next(NodeId::fresh()) + .await? + .ok_or("manual tag event produced no run")?; + assert_eq!(assignment.origin().reference(), "refs/tags/v0.4.0"); + Ok(()) +} + +#[tokio::test] +async fn shadow_mode_persists_the_run_but_never_assigns_it() -> TestResult { + let harness = start_mode(true).await?; + let body = push_body(); + assert_eq!( + deliver(&harness, "repository.ref.updated", &body, true).await?, + 202 + ); + let (_sender, mut inbound) = open_node(&harness).await?; + + assert!( + tokio::time::timeout(Duration::from_millis(100), inbound.next()) + .await + .is_err(), + "shadow mode must not assign the run" + ); + assert!( + harness.runs.take_next(NodeId::fresh()).await?.is_some(), + "the shadow run must still exist in the aggregate" + ); + Ok(()) +} diff --git a/crates/control-node/proto/identity.proto b/crates/control-node/proto/identity.proto --- a/crates/control-node/proto/identity.proto +++ b/crates/control-node/proto/identity.proto @@ -1,60 +1,70 @@ -syntax = "proto3"; - -package syncode.identity.v1; - -service Identity { - rpc ValidateSession(ValidateSessionRequest) returns (ValidateSessionResponse); - rpc CheckCapability(CheckCapabilityRequest) returns (CheckCapabilityResponse); - rpc ResolveRepository(ResolveRepositoryRequest) returns (ResolveRepositoryResponse); -} - -message ResolveRepositoryRequest { - string owner = 1; - string name = 2; -} - -message ResolveRepositoryResponse { - string repository_id = 1; -} - -enum PrincipalKind { - PRINCIPAL_KIND_UNSPECIFIED = 0; - PRINCIPAL_KIND_USER = 1; - PRINCIPAL_KIND_PLATFORM_AGENT = 2; - PRINCIPAL_KIND_LOCAL_AGENT = 3; - PRINCIPAL_KIND_ACCESS_TOKEN = 4; -} - -enum ResourceKind { - RESOURCE_KIND_UNSPECIFIED = 0; - RESOURCE_KIND_ORGANIZATION = 1; - RESOURCE_KIND_REPOSITORY = 2; - RESOURCE_KIND_INSTANCE = 3; -} - -message ValidateSessionRequest { - string session_token = 1; -} - -message ValidateSessionResponse { - string principal_id = 1; - PrincipalKind principal_kind = 2; - int64 expires_at_unix = 3; - string owner_user_id = 4; - repeated string capabilities = 5; - string audience = 6; - ResourceKind resource_kind = 7; - string resource_id = 8; -} - -message CheckCapabilityRequest { - string principal_id = 1; - PrincipalKind principal_kind = 2; - ResourceKind resource_kind = 3; - string resource_id = 4; - string capability = 5; -} - -message CheckCapabilityResponse { - bool allowed = 1; -} +syntax = "proto3"; + +package syncode.identity.v1; + +service Identity { + rpc ValidateSession(ValidateSessionRequest) returns (ValidateSessionResponse); + rpc CheckCapability(CheckCapabilityRequest) returns (CheckCapabilityResponse); + rpc ResolveRepository(ResolveRepositoryRequest) returns (ResolveRepositoryResponse); + rpc GetRepositoryCoordinates(GetRepositoryCoordinatesRequest) returns (GetRepositoryCoordinatesResponse); +} + +message ResolveRepositoryRequest { + string owner = 1; + string name = 2; +} + +message ResolveRepositoryResponse { + string repository_id = 1; +} + +message GetRepositoryCoordinatesRequest { + string repository_id = 1; +} + +message GetRepositoryCoordinatesResponse { + string owner = 1; + string name = 2; +} + +enum PrincipalKind { + PRINCIPAL_KIND_UNSPECIFIED = 0; + PRINCIPAL_KIND_USER = 1; + PRINCIPAL_KIND_PLATFORM_AGENT = 2; + PRINCIPAL_KIND_LOCAL_AGENT = 3; + PRINCIPAL_KIND_ACCESS_TOKEN = 4; +} + +enum ResourceKind { + RESOURCE_KIND_UNSPECIFIED = 0; + RESOURCE_KIND_ORGANIZATION = 1; + RESOURCE_KIND_REPOSITORY = 2; + RESOURCE_KIND_INSTANCE = 3; +} + +message ValidateSessionRequest { + string session_token = 1; +} + +message ValidateSessionResponse { + string principal_id = 1; + PrincipalKind principal_kind = 2; + int64 expires_at_unix = 3; + string owner_user_id = 4; + repeated string capabilities = 5; + string audience = 6; + ResourceKind resource_kind = 7; + string resource_id = 8; +} + +message CheckCapabilityRequest { + string principal_id = 1; + PrincipalKind principal_kind = 2; + ResourceKind resource_kind = 3; + string resource_id = 4; + string capability = 5; +} + +message CheckCapabilityResponse { + bool allowed = 1; +} diff --git a/crates/control-node/src/projection.rs b/crates/control-node/src/projection.rs --- a/crates/control-node/src/projection.rs +++ b/crates/control-node/src/projection.rs @@ -1,59 +1,139 @@ -use syncode_control_runs::ProjectedRun; -use thiserror::Error; -use url::Url; - -/// Pushes a run's projected state to the forge. Dispatch uses it to close the -/// gap between handing a node a token and the forge knowing what that token is -/// for; the periodic sweep in the binary uses it for everything after. -#[derive(Clone)] -pub struct ProjectionClient { - endpoint: Url, - token: String, - client: reqwest::Client, -} - -#[derive(Debug, Error)] -pub enum ProjectionClientError { - #[error("cannot address the projection endpoint")] - Address, -} - -#[derive(Debug, Error)] -pub enum ProjectionRequestError { - #[error(transparent)] - Transport(#[from] reqwest::Error), - #[error("HTTP {status}: {body}")] - Rejected { - status: reqwest::StatusCode, - body: String, - }, -} - -impl ProjectionClient { - pub fn new(base: Url, token: String) -> Result { - let endpoint = base - .join("api/internal/actions/syncode/projection") - .map_err(|_| ProjectionClientError::Address)?; - Ok(Self { - endpoint, - token, - client: reqwest::Client::new(), - }) - } - - pub async fn send(&self, projection: &ProjectedRun) -> Result<(), ProjectionRequestError> { - let response = self - .client - .post(self.endpoint.clone()) - .header("X-Gitea-Internal-Auth", format!("Bearer {}", self.token)) - .json(projection) - .send() - .await?; - let status = response.status(); - if status.is_success() { - return Ok(()); - } - let body = response.text().await?; - Err(ProjectionRequestError::Rejected { status, body }) - } -} +use syncode_control_runs::ProjectedRun; +use thiserror::Error; +use tonic::Request; +use tonic::metadata::{Ascii, MetadataValue}; +use tonic::transport::{Channel, Endpoint}; +use url::Url; +use uuid::Uuid; + +use crate::identity_wire::GetRepositoryCoordinatesRequest; +use crate::identity_wire::identity_client::IdentityClient; + +/// Pushes a run's projected state to the forge. Dispatch uses it to close the +/// gap between handing a node a token and the forge knowing what that token is +/// for; the periodic sweep in the binary uses it for everything after. +#[derive(Clone)] +pub struct ProjectionClient { + endpoint: Url, + token: String, + client: reqwest::Client, + identity: IdentityClient, + authorization: MetadataValue, +} + +#[derive(Debug, Error)] +pub enum ProjectionClientError { + #[error("cannot address the projection endpoint")] + Address, + #[error("cannot address the identity endpoint")] + IdentityAddress, + #[error("cannot authorize identity requests")] + IdentityAuthorization, +} + +#[derive(Debug, Error)] +pub enum ProjectionRequestError { + #[error(transparent)] + Transport(#[from] reqwest::Error), + #[error("HTTP {status}: {body}")] + Rejected { + status: reqwest::StatusCode, + body: String, + }, + #[error("identity rejected the repository projection: {0}")] + Identity(#[from] tonic::Status), + #[error("invalid repository projection origin {0:?}")] + InvalidRepository(String), +} + +impl ProjectionClient { + pub fn new( + base: Url, + token: String, + identity_endpoint: String, + identity_shared_secret: String, + ) -> Result { + let endpoint = base + .join("api/internal/actions/syncode/projection") + .map_err(|_| ProjectionClientError::Address)?; + let identity = IdentityClient::new( + Endpoint::from_shared(identity_endpoint) + .map_err(|_| ProjectionClientError::IdentityAddress)? + .connect_lazy(), + ); + let authorization = format!("Bearer {identity_shared_secret}") + .parse() + .map_err(|_| ProjectionClientError::IdentityAuthorization)?; + Ok(Self { + endpoint, + token, + client: reqwest::Client::new(), + identity, + authorization, + }) + } + + pub async fn send(&self, projection: &ProjectedRun) -> Result<(), ProjectionRequestError> { + let projection = self.for_legacy_projection(projection).await?; + let response = self + .client + .post(self.endpoint.clone()) + .header("X-Gitea-Internal-Auth", format!("Bearer {}", self.token)) + .json(&projection) + .send() + .await?; + let status = response.status(); + if status.is_success() { + return Ok(()); + } + let body = response.text().await?; + Err(ProjectionRequestError::Rejected { status, body }) + } + + async fn for_legacy_projection( + &self, + projection: &ProjectedRun, + ) -> Result { + let repository = projection.origin.repository(); + if Uuid::parse_str(repository).is_ok() { + let mut request = Request::new(GetRepositoryCoordinatesRequest { + repository_id: repository.to_owned(), + }); + request + .metadata_mut() + .insert("authorization", self.authorization.clone()); + let coordinates = self + .identity + .clone() + .get_repository_coordinates(request) + .await? + .into_inner(); + if coordinates.owner.is_empty() + || coordinates.name.is_empty() + || coordinates.owner.contains('/') + || coordinates.name.contains('/') + { + return Err(ProjectionRequestError::InvalidRepository(format!( + "{}/{}", + coordinates.owner, coordinates.name + ))); + } + let mut projected = projection.clone(); + projected.origin = projected + .origin + .with_repository(format!("{}/{}", coordinates.owner, coordinates.name)); + return Ok(projected); + } + let Some((owner, name)) = repository.split_once('/') else { + return Err(ProjectionRequestError::InvalidRepository( + repository.to_owned(), + )); + }; + if owner.is_empty() || name.is_empty() || name.contains('/') { + return Err(ProjectionRequestError::InvalidRepository( + repository.to_owned(), + )); + } + Ok(projection.clone()) + } +} diff --git a/crates/control-runs/src/origin.rs b/crates/control-runs/src/origin.rs --- a/crates/control-runs/src/origin.rs +++ b/crates/control-runs/src/origin.rs @@ -1,159 +1,165 @@ -use std::fmt; - -use serde::{Deserialize, Serialize}; - -/// Which run this is, counted from one across the control plane. -/// -/// A plan carries no identity a person would recognise, and a job asked to -/// name itself has nothing else to say. The number is handed out when the run -/// is opened and written down with it, so a restart does not start counting -/// again. -#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] -pub struct RunNumber(u64); - -/// Where a run came from: the repository and commit its plan was compiled -/// from, the reference the event named, and what that event was. -/// -/// A plan says what to do and nothing about what it is being done to, and a -/// node cannot ask a second question once it holds one. So the run remembers -/// where it came from and every assignment carries it. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub struct Origin { - repository: String, - commit: String, - reference: String, - event: String, - /// The workflow file this run was compiled from, as the repository spells - /// it. One commit can answer one event with several workflows, so the file - /// is what tells their runs apart — and what any of them can be lined up - /// against on the legacy control that ran the same commit. - workflow: String, - #[serde(default)] - delivery: Option, - #[serde(default)] - principal: Option, - #[serde(default = "trusted")] - secrets_allowed: bool, -} - -impl RunNumber { - /// The number the first run of an empty control plane is given. Counting - /// starts at one because zero is not a run anyone can refer to. - #[must_use] - pub const fn first() -> Self { - Self(1) - } - - #[must_use] - pub const fn after(self) -> Self { - Self(self.0 + 1) - } - - #[must_use] - pub const fn get(self) -> u64 { - self.0 - } -} - -impl From for RunNumber { - fn from(value: u64) -> Self { - Self(value) - } -} - -impl fmt::Display for RunNumber { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(formatter) - } -} - -impl Origin { - #[must_use] - pub const fn new( - repository: String, - commit: String, - reference: String, - event: String, - workflow: String, - ) -> Self { - Self { - repository, - commit, - reference, - event, - workflow, - delivery: None, - principal: None, - secrets_allowed: true, - } - } - - #[must_use] - pub fn with_delivery(mut self, delivery: Option) -> Self { - self.delivery = delivery; - self - } - - #[must_use] - pub fn with_principal(mut self, principal: Option) -> Self { - self.principal = principal.filter(|value| !value.is_empty()); - self - } - - #[must_use] - pub const fn with_secret_trust(mut self, allowed: bool) -> Self { - self.secrets_allowed = allowed; - self - } - - #[must_use] - pub fn repository(&self) -> &str { - &self.repository - } - - #[must_use] - pub fn commit(&self) -> &str { - &self.commit - } - - #[must_use] - pub fn reference(&self) -> &str { - &self.reference - } - - #[must_use] - pub fn event(&self) -> &str { - &self.event - } - - #[must_use] - pub fn workflow(&self) -> &str { - &self.workflow - } - - #[must_use] - pub fn delivery(&self) -> Option<&str> { - self.delivery.as_deref() - } - - #[must_use] - pub fn principal(&self) -> Option<&str> { - self.principal.as_deref() - } - - #[must_use] - pub const fn secrets_allowed(&self) -> bool { - self.secrets_allowed - } - - #[must_use] - pub fn organization(&self) -> &str { - self.repository - .split_once('/') - .map_or(self.repository.as_str(), |(organization, _)| organization) - } -} - -const fn trusted() -> bool { - true -} +use std::fmt; + +use serde::{Deserialize, Serialize}; + +/// Which run this is, counted from one across the control plane. +/// +/// A plan carries no identity a person would recognise, and a job asked to +/// name itself has nothing else to say. The number is handed out when the run +/// is opened and written down with it, so a restart does not start counting +/// again. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +pub struct RunNumber(u64); + +/// Where a run came from: the repository and commit its plan was compiled +/// from, the reference the event named, and what that event was. +/// +/// A plan says what to do and nothing about what it is being done to, and a +/// node cannot ask a second question once it holds one. So the run remembers +/// where it came from and every assignment carries it. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct Origin { + repository: String, + commit: String, + reference: String, + event: String, + /// The workflow file this run was compiled from, as the repository spells + /// it. One commit can answer one event with several workflows, so the file + /// is what tells their runs apart — and what any of them can be lined up + /// against on the legacy control that ran the same commit. + workflow: String, + #[serde(default)] + delivery: Option, + #[serde(default)] + principal: Option, + #[serde(default = "trusted")] + secrets_allowed: bool, +} + +impl RunNumber { + /// The number the first run of an empty control plane is given. Counting + /// starts at one because zero is not a run anyone can refer to. + #[must_use] + pub const fn first() -> Self { + Self(1) + } + + #[must_use] + pub const fn after(self) -> Self { + Self(self.0 + 1) + } + + #[must_use] + pub const fn get(self) -> u64 { + self.0 + } +} + +impl From for RunNumber { + fn from(value: u64) -> Self { + Self(value) + } +} + +impl fmt::Display for RunNumber { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(formatter) + } +} + +impl Origin { + #[must_use] + pub const fn new( + repository: String, + commit: String, + reference: String, + event: String, + workflow: String, + ) -> Self { + Self { + repository, + commit, + reference, + event, + workflow, + delivery: None, + principal: None, + secrets_allowed: true, + } + } + + #[must_use] + pub fn with_delivery(mut self, delivery: Option) -> Self { + self.delivery = delivery; + self + } + + #[must_use] + pub fn with_repository(mut self, repository: String) -> Self { + self.repository = repository; + self + } + + #[must_use] + pub fn with_principal(mut self, principal: Option) -> Self { + self.principal = principal.filter(|value| !value.is_empty()); + self + } + + #[must_use] + pub const fn with_secret_trust(mut self, allowed: bool) -> Self { + self.secrets_allowed = allowed; + self + } + + #[must_use] + pub fn repository(&self) -> &str { + &self.repository + } + + #[must_use] + pub fn commit(&self) -> &str { + &self.commit + } + + #[must_use] + pub fn reference(&self) -> &str { + &self.reference + } + + #[must_use] + pub fn event(&self) -> &str { + &self.event + } + + #[must_use] + pub fn workflow(&self) -> &str { + &self.workflow + } + + #[must_use] + pub fn delivery(&self) -> Option<&str> { + self.delivery.as_deref() + } + + #[must_use] + pub fn principal(&self) -> Option<&str> { + self.principal.as_deref() + } + + #[must_use] + pub const fn secrets_allowed(&self) -> bool { + self.secrets_allowed + } + + #[must_use] + pub fn organization(&self) -> &str { + self.repository + .split_once('/') + .map_or(self.repository.as_str(), |(organization, _)| organization) + } +} + +const fn trusted() -> bool { + true +} diff --git a/crates/control-node/src/session/server/tests.rs b/crates/control-node/src/session/server/tests.rs --- a/crates/control-node/src/session/server/tests.rs +++ b/crates/control-node/src/session/server/tests.rs @@ -1,139 +1,141 @@ -#![allow(clippy::expect_used, clippy::panic)] - -use std::time::Duration; - -use syncode_control_nodes::{Ephemeral, Lifecycle, Nodes, Scope}; -use syncode_control_runs::{Forgotten, Runs}; -use tokio::sync::mpsc; -use tokio_stream::wrappers::ReceiverStream; - -use super::{NodeSessionServer, run_session}; -use crate::wire::{ - Capabilities, Capacity, Enrol, Heartbeat, Hello, NodeMessage, control_message, node_message, -}; -use crate::{ArtifactTokenAuthority, CapabilityAuthority, ProjectionClient}; - -#[tokio::test] -async fn an_unreadable_client_releases_its_session() { - let runs = Runs::restored(Forgotten::default()).await.expect("runs"); - let nodes = Nodes::restored(Ephemeral::default()).await.expect("nodes"); - let token = nodes - .issue_token(Scope::Instance) - .await - .expect("enrolment token"); - let server = NodeSessionServer::new( - runs, - nodes.clone(), - CapabilityAuthority::new("test-capability-key").expect("capability authority"), - ArtifactTokenAuthority::new("test-artifact-key").expect("artifact authority"), - "http://127.0.0.1:1/".parse().expect("artifact URL"), - ProjectionClient::new( - "http://127.0.0.1:1/".parse().expect("projection URL"), - String::new(), - ) - .expect("projection client"), - ); - let (input, input_receiver) = mpsc::channel(8); - let (output, mut output_receiver) = mpsc::channel(8); - let session = tokio::spawn(async move { - let mut inbound = ReceiverStream::new(input_receiver); - run_session(&mut inbound, &output, server).await - }); - - input - .send(Ok(message( - 1, - node_message::Body::Enrol(Enrol { - token: token.secret().expose().to_owned(), - }), - ))) - .await - .expect("enrol"); - let enrolled = output_receiver - .recv() - .await - .expect("enrolment response") - .expect("enrolment status"); - let control_message::Body::Enrolled(enrolled) = enrolled.body.expect("enrolment body") else { - panic!("expected enrolment"); - }; - let node = enrolled.node.parse().expect("node id"); - - input - .send(Ok(message( - 2, - node_message::Body::Hello(Hello { - node: enrolled.node, - credential: enrolled.credential, - capabilities: Some(capabilities()), - capacity: Some(capacity()), - max_parallel: 1, - }), - ))) - .await - .expect("hello"); - let welcome = output_receiver - .recv() - .await - .expect("welcome response") - .expect("welcome status"); - assert!(matches!( - welcome.body, - Some(control_message::Body::Welcome(_)) - )); - drop(output_receiver); - - input - .send(Ok(message( - 3, - node_message::Body::Heartbeat(Heartbeat { - held: Vec::new(), - capacity: Some(capacity()), - }), - ))) - .await - .expect("heartbeat"); - tokio::time::timeout(Duration::from_secs(1), session) - .await - .expect("session shutdown") - .expect("session task") - .expect("session cleanup"); - assert_eq!( - nodes.lifecycle(node).await.expect("lifecycle"), - Lifecycle::Offline - ); -} - -fn message(sequence: u64, body: node_message::Body) -> NodeMessage { - NodeMessage { - sequence, - message_id: format!("message-{sequence}"), - idempotency_key: format!("message-{sequence}"), - body: Some(body), - } -} - -fn capabilities() -> Capabilities { - Capabilities { - architecture: "amd64".to_owned(), - operating_system: "linux".to_owned(), - container_runtime: "docker".to_owned(), - container_runtime_version: "29.6.2".to_owned(), - cores: 2, - memory_bytes: 8 * 1024 * 1024 * 1024, - labels: vec!["ubuntu-latest".to_owned()], - } -} - -fn capacity() -> Capacity { - Capacity { - build_volume_free_bytes: 60 * 1024 * 1024 * 1024, - layer_store_bytes: 10 * 1024 * 1024 * 1024, - cache_volume_present: true, - cache_volume_total_bytes: 100, - cache_volume_used_bytes: 40, - cache_volume_path: "/var/cache/syncode".to_owned(), - cached_images: Vec::new(), - cached_actions: Vec::new(), - } -} +#![allow(clippy::expect_used, clippy::panic)] + +use std::time::Duration; + +use syncode_control_nodes::{Ephemeral, Lifecycle, Nodes, Scope}; +use syncode_control_runs::{Forgotten, Runs}; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; + +use super::{NodeSessionServer, run_session}; +use crate::wire::{ + Capabilities, Capacity, Enrol, Heartbeat, Hello, NodeMessage, control_message, node_message, +}; +use crate::{ArtifactTokenAuthority, CapabilityAuthority, ProjectionClient}; + +#[tokio::test] +async fn an_unreadable_client_releases_its_session() { + let runs = Runs::restored(Forgotten::default()).await.expect("runs"); + let nodes = Nodes::restored(Ephemeral::default()).await.expect("nodes"); + let token = nodes + .issue_token(Scope::Instance) + .await + .expect("enrolment token"); + let server = NodeSessionServer::new( + runs, + nodes.clone(), + CapabilityAuthority::new("test-capability-key").expect("capability authority"), + ArtifactTokenAuthority::new("test-artifact-key").expect("artifact authority"), + "http://127.0.0.1:1/".parse().expect("artifact URL"), + ProjectionClient::new( + "http://127.0.0.1:1/".parse().expect("projection URL"), + String::new(), + "http://127.0.0.1:1".to_owned(), + String::new(), + ) + .expect("projection client"), + ); + let (input, input_receiver) = mpsc::channel(8); + let (output, mut output_receiver) = mpsc::channel(8); + let session = tokio::spawn(async move { + let mut inbound = ReceiverStream::new(input_receiver); + run_session(&mut inbound, &output, server).await + }); + + input + .send(Ok(message( + 1, + node_message::Body::Enrol(Enrol { + token: token.secret().expose().to_owned(), + }), + ))) + .await + .expect("enrol"); + let enrolled = output_receiver + .recv() + .await + .expect("enrolment response") + .expect("enrolment status"); + let control_message::Body::Enrolled(enrolled) = enrolled.body.expect("enrolment body") else { + panic!("expected enrolment"); + }; + let node = enrolled.node.parse().expect("node id"); + + input + .send(Ok(message( + 2, + node_message::Body::Hello(Hello { + node: enrolled.node, + credential: enrolled.credential, + capabilities: Some(capabilities()), + capacity: Some(capacity()), + max_parallel: 1, + }), + ))) + .await + .expect("hello"); + let welcome = output_receiver + .recv() + .await + .expect("welcome response") + .expect("welcome status"); + assert!(matches!( + welcome.body, + Some(control_message::Body::Welcome(_)) + )); + drop(output_receiver); + + input + .send(Ok(message( + 3, + node_message::Body::Heartbeat(Heartbeat { + held: Vec::new(), + capacity: Some(capacity()), + }), + ))) + .await + .expect("heartbeat"); + tokio::time::timeout(Duration::from_secs(1), session) + .await + .expect("session shutdown") + .expect("session task") + .expect("session cleanup"); + assert_eq!( + nodes.lifecycle(node).await.expect("lifecycle"), + Lifecycle::Offline + ); +} + +fn message(sequence: u64, body: node_message::Body) -> NodeMessage { + NodeMessage { + sequence, + message_id: format!("message-{sequence}"), + idempotency_key: format!("message-{sequence}"), + body: Some(body), + } +} + +fn capabilities() -> Capabilities { + Capabilities { + architecture: "amd64".to_owned(), + operating_system: "linux".to_owned(), + container_runtime: "docker".to_owned(), + container_runtime_version: "29.6.2".to_owned(), + cores: 2, + memory_bytes: 8 * 1024 * 1024 * 1024, + labels: vec!["ubuntu-latest".to_owned()], + } +} + +fn capacity() -> Capacity { + Capacity { + build_volume_free_bytes: 60 * 1024 * 1024 * 1024, + layer_store_bytes: 10 * 1024 * 1024 * 1024, + cache_volume_present: true, + cache_volume_total_bytes: 100, + cache_volume_used_bytes: 40, + cache_volume_path: "/var/cache/syncode".to_owned(), + cached_images: Vec::new(), + cached_actions: Vec::new(), + } +} diff --git a/tests/projection_client.rs b/tests/projection_client.rs --- /dev/null +++ b/tests/projection_client.rs @@ -1,0 +1,124 @@ +#![allow(clippy::expect_used)] + +use std::error::Error; + +use axum::Json; +use axum::http::StatusCode; +use axum::routing::post; +use syncode_control_node::ProjectionClient; +use syncode_control_node::identity_wire::identity_server::{Identity, IdentityServer}; +use syncode_control_node::identity_wire::{ + CheckCapabilityRequest, CheckCapabilityResponse, GetRepositoryCoordinatesRequest, + GetRepositoryCoordinatesResponse, ResolveRepositoryRequest, ResolveRepositoryResponse, + ValidateSessionRequest, ValidateSessionResponse, +}; +use syncode_control_runs::{Forgotten, JobId, Origin, Runs}; +use tokio::net::TcpListener; +use tokio::sync::mpsc; +use tokio_stream::wrappers::TcpListenerStream; +use tonic::transport::Server; +use tonic::{Request, Response, Status}; + +type TestResult = Result>; + +struct FixtureIdentity; + +#[tonic::async_trait] +impl Identity for FixtureIdentity { + async fn validate_session( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("validate_session")) + } + + async fn check_capability( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("check_capability")) + } + + async fn resolve_repository( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("resolve_repository")) + } + + async fn get_repository_coordinates( + &self, + request: Request, + ) -> Result, Status> { + if request + .metadata() + .get("authorization") + .and_then(|value| value.to_str().ok()) + != Some("Bearer shared-secret") + { + return Err(Status::unauthenticated("missing authorization")); + } + Ok(Response::new(GetRepositoryCoordinatesResponse { + owner: "syncode".to_owned(), + name: "pipelines-demo".to_owned(), + })) + } +} + +#[tokio::test] +async fn native_repository_runs_are_projected_with_legacy_coordinates() -> TestResult { + let identity_listener = TcpListener::bind("127.0.0.1:0").await?; + let identity_endpoint = format!("http://{}", identity_listener.local_addr()?); + tokio::spawn(async move { + let _ = Server::builder() + .add_service(IdentityServer::new(FixtureIdentity)) + .serve_with_incoming(TcpListenerStream::new(identity_listener)) + .await; + }); + + let projection_listener = TcpListener::bind("127.0.0.1:0").await?; + let projection_base = format!("http://{}/", projection_listener.local_addr()?).parse()?; + let (sent, mut received) = mpsc::unbounded_channel(); + let app = axum::Router::new().route( + "/api/internal/actions/syncode/projection", + post(move |Json(body): Json| { + let sent = sent.clone(); + async move { + let _ = sent.send(body); + StatusCode::NO_CONTENT + } + }), + ); + tokio::spawn(async move { + let _ = axum::serve(projection_listener, app).await; + }); + + let runs = Runs::restored(Forgotten::default()).await?; + let run = runs + .queue( + JobId::fresh(), + Origin::new( + "76128383-1df5-4979-9b13-c048a5287e9a".to_owned(), + "commit".to_owned(), + "refs/heads/main".to_owned(), + "push".to_owned(), + ".gitea/workflows/ci.yml".to_owned(), + ) + .with_delivery(Some("delivery".to_owned())), + b"plan".to_vec(), + ) + .await?; + let projection = runs.projection(run).await?.expect("projection"); + ProjectionClient::new( + projection_base, + String::new(), + identity_endpoint, + "shared-secret".to_owned(), + )? + .send(&projection) + .await?; + + let body = received.recv().await.expect("projection body"); + assert_eq!(body["origin"]["repository"], "syncode/pipelines-demo"); + Ok(()) +}