Skip to content
This repository was archived by the owner on Nov 15, 2023. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
139 commits
Select commit Hold shift + click to select a range
0f6c6c1
outgoing messages
drahnr Apr 12, 2022
8fda133
refactor
drahnr Apr 14, 2022
c23c59b
fmt
drahnr Apr 14, 2022
510bed6
better check if missing send
drahnr Apr 19, 2022
2a49f94
fix failing tests
drahnr Apr 19, 2022
d7cab79
impro
drahnr Apr 19, 2022
63cfb62
intermediate commit, use petgraph for convenience and the dot graph
drahnr Apr 19, 2022
6306db8
cleanup overseer a bit, make dummy work
drahnr Apr 20, 2022
9053e17
more migration bits
drahnr Apr 20, 2022
d92a83e
fixups
drahnr Apr 21, 2022
f3ad18d
fixup sample
drahnr Apr 21, 2022
6cb6540
grunt
drahnr Apr 21, 2022
b91630e
fixup pvf-checker
drahnr Apr 22, 2022
ebadddf
more collector traits
drahnr Apr 22, 2022
92e4219
fixins
drahnr Apr 22, 2022
18c027a
move to new generated trait bounds, not compiling yet
drahnr Apr 22, 2022
1e9e46b
fixup disputes-coordinator
drahnr Apr 22, 2022
1cc4e4e
fixup approval voting
drahnr Apr 22, 2022
1552972
fixup availability distribution
drahnr Apr 22, 2022
2b1ca3a
the job based subsystem does not work yet, since the generated trait …
drahnr Apr 22, 2022
0a10c0a
prep deprecation notice
drahnr Apr 22, 2022
9bc9f9d
fixup candidate validation
drahnr Apr 22, 2022
15d2097
update required outbound messages for the overseer
drahnr Apr 22, 2022
0ed6f61
fixup subsystem util
drahnr Apr 22, 2022
11e77e2
half fix, dispatch_iter is there to make things efficient, but uses A…
drahnr Apr 22, 2022
fd1d835
fixup dispute distribution
drahnr Apr 22, 2022
0574a78
more outgoing
drahnr Apr 22, 2022
6e615c7
fixup collator protocol
drahnr Apr 22, 2022
7c5bf06
fixup collator proto
drahnr Apr 22, 2022
3ada1c0
prep provisioner, also job based and hence needs more work
drahnr Apr 22, 2022
11c2b59
chore trait based job sysmte works
drahnr Apr 25, 2022
a4cfe4e
Revert "chore trait based job sysmte works"
drahnr Apr 25, 2022
11eeeb0
minor foo
drahnr Apr 25, 2022
6e6570e
rename subsyste util
drahnr Apr 25, 2022
0503e61
fixup collation
drahnr Apr 25, 2022
4e160c1
fixup provisioner (first job based fixup)
drahnr Apr 25, 2022
2905c2e
update overseer ougoing collation gen
drahnr Apr 25, 2022
cf7ed17
chore
drahnr Apr 27, 2022
1052794
fixup approval voting
drahnr Apr 27, 2022
5f0b3cc
fixup fuckup fixup
drahnr Apr 27, 2022
2fa17c7
avail dist fixup
drahnr Apr 27, 2022
ce4aacd
avail rec + bitfield dist fixup
drahnr Apr 27, 2022
e10c90a
gossip support fixup
drahnr Apr 27, 2022
df9a95a
statement dist fixup
drahnr Apr 27, 2022
e068fc2
fixup annotations overseer
drahnr Apr 27, 2022
118f9ae
fixup borked revert of ToJob -> COnsumes -> Tojob
drahnr Apr 27, 2022
50cbe82
minor fixins
drahnr Apr 27, 2022
b1e3829
split metrics from bridges
drahnr Apr 27, 2022
8950b5f
FIXME FIXME TODO XXX networking bridge hacks
drahnr Apr 27, 2022
06555b7
fixup approval voting
drahnr Apr 27, 2022
27c59ab
chore
drahnr Apr 27, 2022
e199b15
fixup approval dist
drahnr Apr 27, 2022
d17dabc
fixup network bridge
drahnr Apr 27, 2022
a2a6f19
brige: can't leak private type
drahnr Apr 27, 2022
dbf595c
make all Senders send
drahnr Apr 27, 2022
f39bfb9
overseer annotation
drahnr Apr 27, 2022
02053ab
rolling session window explicitness
drahnr Apr 27, 2022
14a23ba
chore: fmt
drahnr May 11, 2022
d6ef62d
fixins
drahnr Apr 27, 2022
fb161d4
minor gimps
drahnr Apr 27, 2022
daf5f6e
avoid impl Trait, use explicit generics
drahnr Apr 28, 2022
9351330
disable type Sender = overseer::OverseerSender
drahnr Apr 28, 2022
bb69d2a
chore: silence some warnings
drahnr Apr 28, 2022
592acd3
add additional `<Context as *ContextTrait>::Sender` bounds
drahnr Apr 28, 2022
bcb08b8
chore fixes
drahnr Apr 29, 2022
41b59ee
overseer-gen: split tests and misc, for readability
drahnr Apr 29, 2022
7e39988
make task name a true literal
drahnr Apr 29, 2022
0c6ce51
make rustc hang
drahnr Apr 29, 2022
d284bc7
chore: move solo up
drahnr Apr 29, 2022
11f8e82
better trait bounds
drahnr Apr 29, 2022
1ef4b79
introduce `#[subsystem]` annotation
drahnr Apr 29, 2022
899f123
parse attr of `#[subsystem]`
drahnr Apr 29, 2022
580964c
drop all dispatch_iter logic
drahnr Apr 29, 2022
18a31f2
feat: use new `#[subsystem]` everywhere
drahnr Apr 29, 2022
2c151d5
chore: fmt
drahnr Apr 29, 2022
5c926a7
fixup overseer gen Ctx sending foo
drahnr Apr 29, 2022
cf8f65d
remove `no_dispatch` from parser
drahnr Apr 29, 2022
40aab17
remove dead extern_network_ty, it's unused
drahnr Apr 29, 2022
422e155
overseer test
drahnr Apr 29, 2022
7e8ecf6
chore: make test cases for overseer more realistic
drahnr May 2, 2022
01f4fff
fix misgeneration, `\#outgoing_wrapper` is not be prefixed with `\#su…
drahnr May 2, 2022
c601dcc
make overseer stuff prep
drahnr May 2, 2022
d5241f7
chore fmt
drahnr May 2, 2022
8132e17
make it all work together
drahnr May 2, 2022
d1d8451
compilation fixes (incomplete)
drahnr May 2, 2022
20c5726
make `#[subsystem]` annotations work across multiple fiels
drahnr May 4, 2022
b9bbe59
fix imports
drahnr May 4, 2022
ce49525
account
drahnr May 4, 2022
3ce7b20
start killing all the stupid trait bounds that are generated
drahnr May 4, 2022
56e003b
add docs to subsystem generation
drahnr May 4, 2022
225cd7a
headlines
drahnr May 4, 2022
5519fd7
fixup backing after rebase
drahnr May 4, 2022
ee5db10
use more ``#[overseer::contextbounds(*, prefix = self::overseer)]``
drahnr May 4, 2022
8972b69
use `#[overseer::contextbounds(..)]` almsot everywhere
drahnr May 4, 2022
b3192ae
polkadot compiles now
drahnr May 4, 2022
031ad5b
foo
drahnr May 5, 2022
17de992
interceptor WIP fixup
drahnr May 5, 2022
d47734d
fix warnings
drahnr May 6, 2022
d3956ae
fix interceptor
drahnr May 6, 2022
99c3c77
fix malus compilation
drahnr May 6, 2022
ea5540a
fix examples
drahnr May 6, 2022
48f4364
remove foo
drahnr May 6, 2022
026b20d
crazy unit type impl
drahnr May 6, 2022
224561c
test fixes
drahnr May 9, 2022
3dba834
example fix
drahnr May 9, 2022
b410971
avoid duplicate code generation
drahnr May 9, 2022
1042794
subsystem util
drahnr May 9, 2022
151c8e7
fix mut
drahnr May 9, 2022
a8a89d7
approval voting test fixes
drahnr May 9, 2022
4d80c53
fix overseer
drahnr May 9, 2022
3e70feb
fix rolling session window and other utils
drahnr May 9, 2022
714b045
undo change for util test
drahnr May 9, 2022
19a8263
foo fmt
drahnr May 9, 2022
cfdfcff
fix network bridge test cases
drahnr May 9, 2022
286db5e
chore: spellcheck
drahnr May 9, 2022
dbcc8b9
fix docs
drahnr May 9, 2022
d6754b5
rebase fallout
drahnr May 10, 2022
727dfcc
fixins
drahnr May 10, 2022
1b93941
cannot format generated files, since we still require nightly fmt
drahnr May 10, 2022
cf59239
add explaining comment
drahnr May 10, 2022
eafd859
please cargo spellcheck
drahnr May 10, 2022
9b32305
docs
drahnr May 10, 2022
80b86e5
do not attempt to compile expansion examples which are not fully corr…
drahnr May 11, 2022
cfb77e9
make doc tests work
drahnr May 11, 2022
f473178
can't teach hunspell `clonabl`, so drop that
drahnr May 11, 2022
0b90f2c
fix typo, causing a rustc panic
drahnr May 11, 2022
e79313b
chore: doc fixes
drahnr May 11, 2022
42fa7be
dev deps cause a false positive for deny(unused_crate_dependencies)
drahnr May 11, 2022
68b4621
warnings
drahnr May 11, 2022
ac2a89d
fix rustdoc complaint
drahnr May 12, 2022
786965b
disable doctests, broken in current nightly
drahnr May 12, 2022
0ba7af4
remove obsolete comment
drahnr May 12, 2022
7db9f33
remove some stale or known todos that don't really affect anything
drahnr May 12, 2022
863adfd
Update node/overseer/overseer-gen/proc-macro/src/impl_subsystem_ctx_s…
drahnr May 12, 2022
4811703
Update node/overseer/overseer-gen/proc-macro/src/parse/parse_subsyste…
drahnr May 12, 2022
0c1093f
Update node/network/bridge/src/metrics.rs
drahnr May 12, 2022
56aff7c
Update node/overseer/overseer-gen/proc-macro/src/subsystem.rs
drahnr May 12, 2022
8f76d68
improve README.md on what subsystem and contextbounds proc macros do
drahnr May 12, 2022
a4ab390
Update node/overseer/overseer-gen/proc-macro/src/overseer.rs
drahnr May 12, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fixup borked revert of ToJob -> COnsumes -> Tojob
  • Loading branch information
drahnr committed May 12, 2022
commit 118f9aeafb10c72fa57b9bb1e9f5f3ea81af8f29
40 changes: 20 additions & 20 deletions node/subsystem-util/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -484,10 +484,10 @@ impl fmt::Debug for FromJobCommand {
/// Other messages are passed along to and from the job via the overseer to other subsystems.
pub trait JobTrait: Unpin + Sized {
/// Message type used to send messages to the job.
type Consumes: 'static + BoundToRelayParent + Send;
type ToJob: 'static + BoundToRelayParent + Send;

/// The set of outgoing messages to be accumalted into.
type Outgoing: 'static + Send;
type OutgoingMessages: 'static + Send;

/// The sender to send outgoing messages.
// The trait bounds are rather minimal.
Expand Down Expand Up @@ -516,7 +516,7 @@ pub trait JobTrait: Unpin + Sized {
leaf: ActivatedLeaf,
run_args: Self::RunArgs,
metrics: Self::Metrics,
receiver: mpsc::Receiver<Self::Consumes>,
receiver: mpsc::Receiver<Self::ToJob>,
sender: JobSender<Self::Sender>,
) -> Pin<Box<dyn Future<Output = Result<(), Self::Error>> + Send>>;
}
Expand All @@ -542,16 +542,16 @@ pub enum JobsError<JobError: std::fmt::Debug + std::error::Error + 'static> {
/// - When dropped, aborts all remaining jobs.
/// - implements `Stream<Item=FromJobCommand>`, collecting all messages from subordinate jobs.
#[pin_project]
struct Jobs<Spawner, Consumes> {
struct Jobs<Spawner, ToJob> {
spawner: Spawner,
running: HashMap<Hash, JobHandle<Consumes>>,
running: HashMap<Hash, JobHandle<ToJob>>,
outgoing_msgs: SelectAll<mpsc::Receiver<FromJobCommand>>,
}

impl<Spawner, Consumes> Jobs<Spawner, Consumes>
impl<Spawner, ToJob> Jobs<Spawner, ToJob>
where
Spawner: SpawnNamed,
Consumes: Send + 'static,
ToJob: Send + 'static,
{
/// Create a new Jobs manager which handles spawning appropriate jobs.
pub fn new(spawner: Spawner) -> Self {
Expand All @@ -566,7 +566,7 @@ where
metrics: Job::Metrics,
sender: Job::Sender,
) where
Job: JobTrait<Consumes = Consumes>,
Job: JobTrait<ToJob = ToJob>,
{
let hash = leaf.hash;
let (to_job_tx, to_job_rx) = mpsc::channel(JOB_CHANNEL_CAPACITY);
Expand Down Expand Up @@ -613,7 +613,7 @@ where
}

/// Send a message to the appropriate job for this `parent_hash`.
async fn send_msg(&mut self, parent_hash: Hash, msg: Consumes) {
async fn send_msg(&mut self, parent_hash: Hash, msg: ToJob) {
if let Entry::Occupied(mut job) = self.running.entry(parent_hash) {
if job.get_mut().send_msg(msg).await.is_err() {
job.remove();
Expand All @@ -622,7 +622,7 @@ where
}
}

impl<Spawner, Consumes> Stream for Jobs<Spawner, Consumes>
impl<Spawner, ToJob> Stream for Jobs<Spawner, ToJob>
where
Spawner: SpawnNamed,
{
Expand All @@ -637,7 +637,7 @@ where
}
}

impl<Spawner, Consumes> stream::FusedStream for Jobs<Spawner, Consumes>
impl<Spawner, ToJob> stream::FusedStream for Jobs<Spawner, ToJob>
where
Spawner: SpawnNamed,
{
Expand All @@ -661,7 +661,7 @@ pub struct JobSubsystemParams<Spawner, RunArgs, Metrics> {
/// Conceptually, this is very simple: it just loops forever.
///
/// - On incoming overseer messages, it starts or stops jobs as appropriate.
/// - On other incoming messages, if they can be converted into `Job::Consumes` and
/// - On other incoming messages, if they can be converted into `Job::ToJob` and
/// include a hash, then they're forwarded to the appropriate individual job.
/// - On outgoing messages from the jobs, it forwards them to the overseer.
pub struct JobSubsystem<Job: JobTrait, Spawner> {
Expand All @@ -684,20 +684,20 @@ impl<Job: JobTrait, Spawner> JobSubsystem<Job, Spawner> {
where
Spawner: SpawnNamed + Send + Clone + Unpin + 'static,
Context: SubsystemContext<
Message = <Job as JobTrait>::Consumes,
OutgoingMessages = <Job as JobTrait>::Outgoing,
Signal = OverseerSignal,
Message = <Job as JobTrait>::ToJob,
OutgoingMessages = <Job as JobTrait>::OutgoingMessages,
Sender = <Job as JobTrait>::Sender,
Signal = OverseerSignal,
>,
Job: 'static + JobTrait + Send,
<Job as JobTrait>::RunArgs: Clone + Sync,
<Job as JobTrait>::Consumes:
<Job as JobTrait>::ToJob:
Sync + From<<Context as polkadot_overseer::SubsystemContext>::Message>,
<Job as JobTrait>::Metrics: Sync,
{
let JobSubsystem { params: JobSubsystemParams { spawner, run_args, metrics }, .. } = self;

let mut jobs = Jobs::<Spawner, Job::Consumes>::new(spawner);
let mut jobs = Jobs::<Spawner, Job::ToJob>::new(spawner);

loop {
select! {
Expand Down Expand Up @@ -764,14 +764,14 @@ impl<Context, Job, Spawner> Subsystem<Context, SubsystemError> for JobSubsystem<
where
Spawner: SpawnNamed + Send + Clone + Unpin + 'static,
Context: SubsystemContext<
Message = Job::Consumes,
Message = Job::ToJob,
Signal = OverseerSignal,
OutgoingMessages = <Job as JobTrait>::Outgoing,
OutgoingMessages = <Job as JobTrait>::OutgoingMessages,
Sender = <Job as JobTrait>::Sender,
>,
Job: 'static + JobTrait + Send,
Job::RunArgs: Clone + Sync,
<Job as JobTrait>::Consumes: Sync + From<<Context as SubsystemContext>::Message>,
<Job as JobTrait>::ToJob: Sync + From<<Context as SubsystemContext>::Message>,
Job::Metrics: Sync,
{
fn start(self, ctx: Context) -> SpawnedSubsystem {
Expand Down
27 changes: 19 additions & 8 deletions node/subsystem-util/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,9 @@ use thiserror::Error;

// job structs are constructed within JobTrait::run
// most will want to retain the sender and receiver, as well as whatever other data they like
struct FakeCollatorProtocolJob {
struct FakeCollatorProtocolJob<Sender> {
receiver: mpsc::Receiver<CollatorProtocolMessage>,
_phantom: std::marker::PhantomData<Sender>,
}

// Error will mostly be a wrapper to make the try operator more convenient;
Expand All @@ -57,12 +58,13 @@ enum Error {
Sending(#[from] mpsc::SendError),
}

impl<Sender> JobTrait for FakeCollatorProtocolJob
impl<Sender> JobTrait for FakeCollatorProtocolJob<Sender>
where
Sender: overseer::CollatorProtocolSender,
{
type Consumes = CollatorProtocolMessage;
type Outgoing = CollatorProtocolOutgoingMessages;
type OutgoingMessages = CollatorProtocolOutgoingMessages;
type Sender = Sender;
type Error = Error;
type RunArgs = bool;
type Metrics = ();
Expand All @@ -72,20 +74,23 @@ where
/// Run a job for the parent block indicated
//
// this function is in charge of creating and executing the job's main loop
fn run<S: overseer::CollatorProtocolSender>(
fn run(
_: ActivatedLeaf,
run_args: Self::RunArgs,
_metrics: Self::Metrics,
receiver: mpsc::Receiver<CollatorProtocolMessage>,
mut sender: JobSender<S>,
mut sender: JobSender<Sender>,
) -> Pin<Box<dyn Future<Output = Result<(), Self::Error>> + Send>> {
async move {
let job = FakeCollatorProtocolJob { receiver };
let job = FakeCollatorProtocolJob {
receiver,
_phantom: std::marker::PhantomData::<Sender>,
};

if run_args {
sender
.send_message(CollatorProtocolMessage::Invalid(
Default::default(),
dummy_hash(),
dummy_candidate_receipt(dummy_hash()),
))
.await;
Expand Down Expand Up @@ -115,7 +120,13 @@ impl FakeCollatorProtocolJob {
}

// with the job defined, it's straightforward to get a subsystem implementation.
type FakeCollatorProtocolSubsystem<Spawner> = JobSubsystem<FakeCollatorProtocolJob, Spawner>;
type FakeCollatorProtocolSubsystem<Spawner> =
JobSubsystem<
FakeCollatorProtocolJob<
test_helpers::TestSubsystemSender
>,
Spawner,
>;

// this type lets us pretend to be the overseer
type OverseerHandle = test_helpers::TestSubsystemContextHandle<CollatorProtocolMessage>;
Expand Down