diff --git a/crates/oq3_parser/src/input.rs b/crates/oq3_parser/src/input.rs index d6664ad..0812f5d 100644 --- a/crates/oq3_parser/src/input.rs +++ b/crates/oq3_parser/src/input.rs @@ -15,7 +15,7 @@ type bits = u64; /// /// As of now, parser doesn't have access to the *text* of the tokens, and makes /// decisions based solely on their classification. Unlike `LexerToken`, the -/// `Tokens` doesn't include whitespace and comments. Main input to the parser. +/// `Token` doesn't include whitespace and comments. Main input to the parser. /// /// Struct of arrays internally, but this shouldn't really matter. #[derive(Default)] @@ -25,19 +25,10 @@ pub struct Input { /// Account for whitespace/comments dropped on construction joint: Vec, - contextual_kind: Vec, } /// `pub` impl used by callers to create `Tokens`. impl Input { - #[inline] - pub fn push(&mut self, kind: SyntaxKind) { - self.push_impl(kind, SyntaxKind::EOF) - } - #[inline] - pub fn push_ident(&mut self, contextual_kind: SyntaxKind) { - self.push_impl(SyntaxKind::IDENT, contextual_kind) - } /// Sets jointness for the last token we've pushed. /// /// This is a separate API rather than an argument to the `push` to make it @@ -61,13 +52,12 @@ impl Input { self.joint[idx] |= 1 << b_idx; } #[inline] - fn push_impl(&mut self, kind: SyntaxKind, contextual_kind: SyntaxKind) { + pub fn push(&mut self, kind: SyntaxKind) { let idx = self.len(); if idx % (bits::BITS as usize) == 0 { self.joint.push(0); } self.kind.push(kind); - self.contextual_kind.push(contextual_kind); } } @@ -76,9 +66,7 @@ impl Input { pub(crate) fn kind(&self, idx: usize) -> SyntaxKind { self.kind.get(idx).copied().unwrap_or(SyntaxKind::EOF) } - // pub(crate) fn contextual_kind(&self, idx: usize) -> SyntaxKind { - // self.contextual_kind.get(idx).copied().unwrap_or(SyntaxKind::EOF) - // } + pub(crate) fn is_joint(&self, n: usize) -> bool { let (idx, b_idx) = self.bit_index(n); self.joint[idx] & (1 << b_idx) != 0 diff --git a/crates/oq3_parser/src/parser.rs b/crates/oq3_parser/src/parser.rs index 0d3f260..580cee2 100644 --- a/crates/oq3_parser/src/parser.rs +++ b/crates/oq3_parser/src/parser.rs @@ -237,21 +237,6 @@ impl<'t> Parser<'t> { // (ends_in_dot, marker) // } - // Unused. OQ3 has no contextual keywords. - // /// Advances the parser by one token, remapping its kind. - // /// This is useful to create contextual keywords from - // /// identifiers. For example, the lexer creates a `union` - // /// *identifier* token, but the parser remaps it to the - // /// `union` keyword, and keyword is what ends up in the - // /// final tree. - // pub(crate) fn _bump_remap(&mut self, kind: SyntaxKind) { - // if self.current() == EOF { - // // FIXME: panic!? - // return; - // } - // self.do_bump(kind, 1); - // } - /// Emit error with the `message`. /// FIXME (not GJL): this should be much more fancy and support /// structured errors with spans and notes, like rustc diff --git a/crates/oq3_parser/src/shortcuts.rs b/crates/oq3_parser/src/shortcuts.rs index c6490a3..9e19ecd 100644 --- a/crates/oq3_parser/src/shortcuts.rs +++ b/crates/oq3_parser/src/shortcuts.rs @@ -38,22 +38,18 @@ impl LexedStr<'_> { // whitespace or comment was_joint = false } else { - if kind == SyntaxKind::IDENT { - let contextual_kw = SyntaxKind::IDENT; - res.push_ident(contextual_kw); - } else { - if was_joint { - res.was_joint(); - } - res.push(kind); - // Tag the token as joint if it is float with a fractional part. - // We use this jointness to inform the parser about what token split - // event to emit when we encounter a float literal in a field access - if kind == SyntaxKind::FLOAT_NUMBER && !self.text(i).ends_with('.') { - res.was_joint(); - } + if was_joint { + res.was_joint(); + } + res.push(kind); + // Tag the token as joint if it is float with a fractional part. + // We use this jointness to inform the parser about what token split + // event to emit when we encounter a float literal in a field access + // TODO: At d0146087 test suite passes with following conditional + // deleted. Is it only that we are not testing for it? + if kind == SyntaxKind::FLOAT_NUMBER && !self.text(i).ends_with('.') { + res.was_joint(); } - was_joint = true; } } diff --git a/crates/oq3_syntax/build.rs b/crates/oq3_syntax/build.rs index 53ea19b..d40f6d8 100644 --- a/crates/oq3_syntax/build.rs +++ b/crates/oq3_syntax/build.rs @@ -900,14 +900,7 @@ mod sourcegen { let full_keywords_values = grammar.keywords; let full_keywords = full_keywords_values.iter().map(upper_snake); - // let contextual_keywords_values = &grammar.contextual_keywords; - // let contextual_keywords = contextual_keywords_values.iter().map(upper_snake); - let all_keywords_values = grammar.keywords.to_vec(); - // .iter() - // // .chain(grammar.contextual_keywords.iter()) - // .copied() - // .collect::>(); let all_keywords_idents = all_keywords_values.iter().map(|kw| format_ident!("{}", kw)); let all_keywords = all_keywords_values .iter() diff --git a/crates/oq3_syntax/openqasm3.ungram b/crates/oq3_syntax/openqasm3.ungram index 4bee3a4..a8ba04a 100644 --- a/crates/oq3_syntax/openqasm3.ungram +++ b/crates/oq3_syntax/openqasm3.ungram @@ -1,6 +1,6 @@ // This grammar specifies the structure of the OpenQASM 3 concrete syntax tree. // It does not specify parsing rules (ambiguities, precedence, etc are out of scope). -// Tokens are processed -- contextual keywords are recognised, compound operators glued. +// Tokens are processed -- compound operators glued. // // Legend: //