Grcov report - tokenizer.rs

1

use alloc::borrow::Cow;

2

use alloc::string::String;

3

use core::fmt::Display;

4

use core::mem;

5

use core::ops::Range;

6

7

use unicode_ident::{is_xid_continue, is_xid_start};

8

9

use crate::tokenizer::char_iterator::CharIterator;

10

11

mod char_iterator;

12

13

/// A token in an Rsn document.

14

#[derive(Clone, Eq, PartialEq, Debug)]

15

pub struct Token<'a> {

16

    /// The byte range of this token.

17

    pub location: Range<usize>,

18

    /// The kind of this token.

19

    pub kind: TokenKind<'a>,

20

21

22

impl<'a> Token<'a> {

23

    #[must_use]

24

1587

    const fn new(location: Range<usize>, kind: TokenKind<'a>) -> Self {

25

1587

        Self { location, kind }

26

1587

27

28

29

/// A kind of a token in an Rsn document.

30

#[derive(Clone, Debug)]

31

pub enum TokenKind<'a> {

32

    /// An integer literal.

33

    Integer(Integer),

34

    /// A floating point literal.

35

    Float(f64),

36

    /// A boolean literal.

37

    Bool(bool),

38

    /// A character literal.

39

    Character(char),

40

    /// The `:` character.

41

    Colon,

42

    /// The `,` character.

43

    Comma,

44

    /// A byte literal.

45

    Byte(u8),

46

    /// A string literal.

47

    String(Cow<'a, str>),

48

    /// A byte string literal.

49

    Bytes(Cow<'a, [u8]>),

50

    /// An identifier (name).

51

    Identifier(&'a str),

52

    /// The opening variant of a [`Balanced`] token.

53

    Open(Balanced),

54

    /// The closing variant of a [`Balanced`] token.

55

    Close(Balanced),

56

    /// A comment

57

    Comment(&'a str),

58

    /// Whitespace between other tokens.

59

    Whitespace(&'a str),

60

61

62

impl<'a> Eq for TokenKind<'a> {}

63

64

impl<'a> PartialEq for TokenKind<'a> {

65

132

    fn eq(&self, other: &Self) -> bool {

66

132

        match (self, other) {

67

58

            (Self::Integer(l0), Self::Integer(r0)) => l0 == r0,

68

20

            (Self::Float(l0), Self::Float(r0)) => l0.total_cmp(r0).is_eq(),

69

2

            (Self::Bool(l0), Self::Bool(r0)) => l0 == r0,

70

8

            (Self::Character(l0), Self::Character(r0)) => l0 == r0,

71

7

            (Self::Byte(l0), Self::Byte(r0)) => l0 == r0,

72

8

            (Self::String(l0), Self::String(r0)) => l0 == r0,

73

13

            (Self::Bytes(l0), Self::Bytes(r0)) => l0 == r0,

74

4

            (Self::Comment(l0), Self::Comment(r0))

75

10

            | (Self::Identifier(l0), Self::Identifier(r0)) => l0 == r0,

76

2

            (Self::Open(l0), Self::Open(r0)) | (Self::Close(l0), Self::Close(r0)) => l0 == r0,

77

4

            _ => core::mem::discriminant(self) == core::mem::discriminant(other),

78

79

132

80

81

82

/// An integer literal.

83

#[derive(Clone, Copy, Eq, PartialEq, Debug)]

84

pub enum Integer {

85

    /// An unsigned integer that fits within an `usize`.

86

    Usize(usize),

87

    /// A signed integer that fits within an `isize`.

88

    Isize(isize),

89

    /// An unsigned integer that is too large to fit within an `usize`.

90

    UnsignedLarge(UnsignedLarge),

91

    /// A signed integer that is too large to fit within an `isize`.

92

    SignedLarge(SignedLarge),

93

94

95

macro_rules! fn_integer_into {

96

    ($name:ident, $type:ty) => {

97

        /// Returns this integer as a

98

        #[doc = stringify!($type)]

99

        /// if the value can fit in a

100

        #[doc = stringify!($type)]

101

        #[inline]

102

        #[must_use]

103

202

        pub fn $name(self) -> Option<$type> {

104

202

            match self {

105

165

                Integer::Usize(value) => value.try_into().ok(),

106

37

                Integer::Isize(value) => value.try_into().ok(),

107

                Integer::UnsignedLarge(value) => value.try_into().ok(),

108

                Integer::SignedLarge(value) => value.try_into().ok(),

109

110

202

111

};

112

113

114

impl Integer {

115

    fn_integer_into!(as_u8, u8);

116

117

    fn_integer_into!(as_u16, u16);

118

119

    fn_integer_into!(as_u32, u32);

120

121

    fn_integer_into!(as_u64, u64);

122

123

    fn_integer_into!(as_u128, u128);

124

125

    fn_integer_into!(as_usize, usize);

126

127

    fn_integer_into!(as_i8, i8);

128

129

    fn_integer_into!(as_i16, i16);

130

131

    fn_integer_into!(as_i32, i32);

132

133

    fn_integer_into!(as_i64, i64);

134

135

    fn_integer_into!(as_i128, i128);

136

137

    fn_integer_into!(as_isize, isize);

138

139

    /// Returns true if this number is equal to 0.

140

    #[inline]

141

    #[must_use]

142

    pub const fn is_zero(self) -> bool {

143

        match self {

144

            Integer::Usize(value) => value == 0,

145

            Integer::Isize(value) => value == 0,

146

            Integer::UnsignedLarge(value) => value == 0,

147

            Integer::SignedLarge(value) => value == 0,

148

149

150

151

    /// Returns this number cast to an `f64`.

152

    #[must_use]

153

    #[allow(clippy::cast_precision_loss)]

154

    pub fn as_f64(self) -> f64 {

155

        match self {

156

            Integer::Usize(value) => value as f64,

157

            Integer::Isize(value) => value as f64,

158

            Integer::UnsignedLarge(value) => value as f64,

159

            Integer::SignedLarge(value) => value as f64,

160

161

162

163

164

#[cfg(feature = "integer128")]

165

type SignedLarge = i128;

166

#[cfg(feature = "integer128")]

167

type UnsignedLarge = u128;

168

169

#[cfg(not(feature = "integer128"))]

170

type SignedLarge = i64;

171

#[cfg(not(feature = "integer128"))]

172

type UnsignedLarge = u64;

173

174

impl From<usize> for Integer {

175

200

    fn from(value: usize) -> Self {

176

200

        Self::Usize(value)

177

200

178

179

180

impl From<isize> for Integer {

181

43

    fn from(value: isize) -> Self {

182

43

        Self::Isize(value)

183

43

184

185

186

macro_rules! impl_from_primitive {

187

    ($variant:ident, $target:ty, $ty:ty) => {

188

        impl From<$ty> for Integer {

189

4

            fn from(value: $ty) -> Self {

190

4

                Self::$variant(<$target>::from(value))

191

4

192

193

};

194

195

196

impl_from_primitive!(Usize, usize, u8);

197

impl_from_primitive!(Usize, usize, u16);

198

impl_from_primitive!(Isize, isize, i8);

199

impl_from_primitive!(Isize, isize, i16);

200

201

macro_rules! impl_from_primitive_with_fallback {

202

    ($variant:ident, $target:ty, $larger:ident, $ty:ty) => {

203

        impl From<$ty> for Integer {

204

6

            fn from(value: $ty) -> Self {

205

6

                match <$target>::try_from(value) {

206

6

                    Ok(value) => Self::$variant(value),

207

                    Err(_) => Self::$larger(<$larger>::from(value)),

208

209

6

210

211

};

212

213

214

impl_from_primitive_with_fallback!(Usize, usize, UnsignedLarge, u32);

215

impl_from_primitive_with_fallback!(Isize, isize, SignedLarge, i32);

216

impl_from_primitive_with_fallback!(Usize, usize, UnsignedLarge, u64);

217

impl_from_primitive_with_fallback!(Isize, isize, SignedLarge, i64);

218

219

macro_rules! impl_try_from_primitive {

220

    ($variant:ident, $target:ty, $larger:ident, $ty:ty) => {

221

        impl TryFrom<$ty> for Integer {

222

            type Error = core::num::TryFromIntError;

223

224

2

            fn try_from(value: $ty) -> Result<Self, Self::Error> {

225

2

                match <$target>::try_from(value) {

226

2

                    Ok(value) => Ok(Self::$variant(value)),

227

                    Err(_) => Ok(Self::$larger(<$larger>::try_from(value)?)),

228

229

2

230

231

};

232

233

234

impl_try_from_primitive!(Usize, usize, UnsignedLarge, u128);

235

impl_try_from_primitive!(Isize, isize, SignedLarge, i128);

236

237

/// A token kind that is expected to have a balanced number of open and closing

238

/// variants.

239

#[derive(Clone, Copy, Eq, PartialEq, Debug)]

240

pub enum Balanced {

241

    /// Parentheses

242

    Paren,

243

    /// Curly Braces

244

    Brace,

245

    /// Square Brackets

246

    Bracket,

247

248

249

/// Parses Rsn into a sequence of [`Token`]s.

250

#[derive(Debug, Clone)]

251

pub struct Tokenizer<'a, const INCLUDE_ALL: bool> {

252

    chars: CharIterator<'a>,

253

    scratch: String,

254

255

256

impl<'a> Tokenizer<'a, false> {

257

    /// Returns a tokenizer that ignores whitespace and comments.

258

    #[must_use]

259

173

    pub fn minified(source: &'a str) -> Self {

260

173

        Self::new(source)

261

173

262

263

264

impl<'a> Tokenizer<'a, true> {

265

    /// Returns a tokenizer that includes whitespace and comments.

266

    #[must_use]

267

6

    pub fn full(source: &'a str) -> Self {

268

6

        Self::new(source)

269

6

270

271

272

impl<'a, const INCLUDE_ALL: bool> Tokenizer<'a, INCLUDE_ALL> {

273

179

    fn new(source: &'a str) -> Self {

274

179

        Self {

275

179

            chars: CharIterator::new(source),

276

179

            scratch: String::new(),

277

179

278

179

279

280

    /// Returns the current byte offset of the tokenizer.

281

    #[must_use]

282

1358

    pub const fn current_offset(&self) -> usize {

283

1358

        self.chars.current_offset()

284

1358

285

286

1821

    fn next_or_eof(&mut self) -> Result<char, Error> {

287

1821

        self.chars

288

1821

            .next()

289

1821

            .ok_or_else(|| self.error_at_last_char(ErrorKind::UnexpectedEof))

290

1821

291

292

    fn error(&self, kind: ErrorKind) -> Error {

293

        Error::new(self.chars.marked_range(), kind)

294

295

296

    fn error_at_last_char(&self, kind: ErrorKind) -> Error {

297

        Error::new(self.chars.last_char_range(), kind)

298

299

300

    fn error_at_next_char(&mut self, kind: ErrorKind) -> Error {

301

        let range = if let Some((offset, ch)) = self.chars.peek_full() {

302

            offset..offset + ch.len_utf8()

303

        } else {

304

            let mut range = self.chars.last_char_range();

305

            range.start = range.end;

306

            range

307

};

308

        Error::new(range, kind)

309

310

311

    #[allow(clippy::cast_possible_truncation)]

312

219

    fn tokenize_positive_integer<I>(&mut self, mut value: I) -> Result<Token<'a>, Error>

313

219

    where

314

219

        I: Integral,

315

219

        Integer: From<I> + TryFrom<I::Larger>,

316

219

317

219

        let mut has_decimal = false;

318

219

        let mut has_exponent = false;

319

219

        let mut had_underscores = false;

320

219

        let mut overflowing = false;

321

877

        while let Some(ch) = self.chars.peek() {

322

865

            if let Some(digit_value) = ch.to_digit(10) {

323

638

                if let Some(new_value) = value

324

638

                    .checked_mul(I::from(10))

325

638

                    .and_then(|value| value.checked_add(I::from(digit_value as u8)))

326

638

327

638

                    value = new_value;

328

638

                    self.chars.next();

329

638

                } else {

330

                    overflowing = true;

331

                    break;

332

333

227

            } else if ch == '.' {

334

11

                has_decimal = true;

335

11

                self.chars.next();

336

11

                break;

337

216

            } else if ch == 'e' || ch == 'E' {

338

3

                has_decimal = true;

339

3

                has_exponent = true;

340

3

                self.chars.next();

341

3

                break;

342

213

            } else if ch == '_' {

343

20

                self.chars.next();

344

20

                had_underscores = true;

345

20

            } else {

346

193

                break;

347

348

349

350

219

        if overflowing {

351

            let mut value: I::Larger = value.into_larger();

352

            while let Some(ch) = self.chars.peek() {

353

                if let Some(digit_value) = ch.to_digit(10) {

354

                    if let Some(new_value) = value

355

                        .checked_mul(<I::Larger>::from(10))

356

                        .and_then(|value| value.checked_add(<I::Larger>::from(digit_value as u8)))

357

358

                        value = new_value;

359

                        self.chars.next();

360

                    } else {

361

                        return Err(self.error(ErrorKind::IntegerTooLarge));

362

363

                } else if ch == '.' {

364

                    has_decimal = true;

365

                    self.chars.next();

366

                    break;

367

                } else if ch == 'e' || ch == 'E' {

368

                    has_decimal = true;

369

                    has_exponent = true;

370

                    self.chars.next();

371

                    break;

372

                } else if ch == '_' {

373

                    self.chars.next();

374

                    had_underscores = true;

375

                } else {

376

                    break;

377

378

379

380

            if !has_decimal {

381

                let integer = Integer::try_from(value).map_err(|_| {

382

                    Error::new(self.chars.marked_range(), ErrorKind::IntegerTooLarge)

383

                })?;

384

                return Ok(Token::new(

385

                    self.chars.marked_range(),

386

                    TokenKind::Integer(integer),

387

));

388

389

219

390

391

219

        if has_decimal {

392

14

            self.tokenize_float(had_underscores, has_exponent)

393

        } else {

394

205

            Ok(Token::new(

395

205

                self.chars.marked_range(),

396

205

                TokenKind::Integer(Integer::from(value)),

397

205

))

398

399

219

400

401

    #[allow(clippy::cast_possible_truncation)]

402

44

    fn tokenize_negative_integer<I>(&mut self, mut value: I) -> Result<Token<'a>, Error>

403

44

    where

404

44

        I: Integral,

405

44

        Integer: From<I> + TryFrom<I::Larger>,

406

44

407

44

        let mut has_decimal = false;

408

44

        let mut has_exponent = false;

409

44

        let mut overflowing = false;

410

44

        let mut had_underscores = false;

411

364

        while let Some(ch) = self.chars.peek() {

412

356

            if let Some(digit_value) = ch.to_digit(10) {

413

310

                if let Some(new_value) = value

414

310

                    .checked_mul(I::from(10))

415

310

                    .and_then(|value| value.checked_sub(I::from(digit_value as u8)))

416

310

417

310

                    value = new_value;

418

310

                    self.chars.next();

419

310

                } else {

420

                    overflowing = true;

421

                    break;

422

423

46

            } else if ch == '.' {

424

5

                has_decimal = true;

425

5

                self.chars.next();

426

5

                break;

427

41

            } else if ch == 'e' || ch == 'E' {

428

1

                has_decimal = true;

429

1

                has_exponent = true;

430

1

                self.chars.next();

431

1

                break;

432

40

            } else if ch == '_' {

433

10

                had_underscores = true;

434

10

                self.chars.next();

435

10

            } else {

436

30

                break;

437

438

439

440

44

        if overflowing {

441

            let mut value: I::Larger = value.into_larger();

442

            while let Some(ch) = self.chars.peek() {

443

                if let Some(digit_value) = ch.to_digit(10) {

444

                    if let Some(new_value) = value

445

                        .checked_mul(<I::Larger>::from(10))

446

                        .and_then(|value| value.checked_sub(<I::Larger>::from(digit_value as u8)))

447

448

                        value = new_value;

449

                        self.chars.next();

450

                    } else {

451

                        return Err(self.error(ErrorKind::IntegerTooLarge));

452

453

                } else if ch == '.' {

454

                    has_decimal = true;

455

                    self.chars.next();

456

                    break;

457

                } else if ch == 'e' || ch == 'E' {

458

                    has_decimal = true;

459

                    has_exponent = true;

460

                    self.chars.next();

461

                    break;

462

                } else if ch == '_' {

463

                    had_underscores = true;

464

                    self.chars.next();

465

                } else {

466

                    break;

467

468

469

470

            if !has_decimal {

471

                let integer = Integer::try_from(value).map_err(|_| {

472

                    Error::new(self.chars.marked_range(), ErrorKind::IntegerTooLarge)

473

                })?;

474

                return Ok(Token::new(

475

                    self.chars.marked_range(),

476

                    TokenKind::Integer(integer),

477

));

478

479

44

480

481

44

        if has_decimal {

482

6

            self.tokenize_float(had_underscores, has_exponent)

483

        } else {

484

38

            Ok(Token::new(

485

38

                self.chars.marked_range(),

486

38

                TokenKind::Integer(Integer::from(value)),

487

38

))

488

489

44

490

491

20

    fn tokenize_float(

492

20

        &mut self,

493

20

        had_underscores: bool,

494

20

        mut has_exponent: bool,

495

20

    ) -> Result<Token<'a>, Error> {

496

20

        self.scratch.clear();

497

20

        let already_read_chars = self.chars.marked_str();

498

20

        if had_underscores {

499

            self.scratch

500

                .extend(already_read_chars.chars().filter(|ch| ch != &'_'));

501

20

        } else {

502

20

            self.scratch.push_str(already_read_chars);

503

20

504

505

20

        if !has_exponent {

506

            // Read any decimal digits

507

31

            while let Some(ch) = self.chars.peek() {

508

27

                if ch.is_ascii_digit() {

509

15

                    self.scratch.push(ch);

510

15

                    self.chars.next();

511

15

                } else if !has_exponent && ch == 'e' || ch == 'E' {

512

8

                    self.scratch.push(ch);

513

8

                    has_exponent = true;

514

8

                    self.chars.next();

515

8

516

8

                    break;

517

                } else {

518

4

                    break;

519

520

521

4

522

523

20

        if has_exponent {

524

            // Handle the exponent sign

525

12

            if let Some(ch) = self.chars.peek() {

526

12

                if ch == '+' || ch == '-' {

527

4

                    self.scratch.push(ch);

528

4

                    self.chars.next();

529

8

530

531

532

            // Require at least one digit for the exponent, but allow

533

            // skipping underscores.

534

12

            let mut has_exponent_digit = false;

535

30

            while let Some(ch) = self.chars.peek() {

536

20

                let is_digit = ch.is_ascii_digit();

537

20

538

20

                if is_digit || ch == '_' {

539

18

                    has_exponent_digit |= is_digit;

540

18

                    self.scratch.push(ch);

541

18

                    self.chars.next();

542

18

                } else {

543

2

                    break;

544

545

546

547

12

            if !has_exponent_digit {

548

                return Err(self.error_at_next_char(ErrorKind::ExpectedDigit));

549

12

550

8

551

552

20

        let parsed = self

553

20

            .scratch

554

20

            .parse::<f64>()

555

20

            .map_err(|_| self.error(ErrorKind::InvalidFloat))?;

556

557

20

        Ok(Token::new(

558

20

            self.chars.marked_range(),

559

20

            TokenKind::Float(parsed),

560

20

))

561

20

562

563

    fn tokenize_radix_large_number<const BITS: u32>(

564

        &mut self,

565

        signed: bool,

566

        negative: bool,

567

        value: usize,

568

        first_hex_value: u32,

569

    ) -> Result<Token<'a>, Error> {

570

        assert!(BITS == 1 || BITS == 3 || BITS == 4);

571

        let max = 2_u8.pow(BITS);

572

        let mut value = value as UnsignedLarge;

573

        value <<= BITS;

574

        value |= first_hex_value as UnsignedLarge;

575

576

        while let Some(result) = self.chars.peek().map(|ch| ch.to_digit(BITS * 4).ok_or(ch)) {

577

            match result {

578

                Ok(radix_value) => {

579

                    self.chars.next();

580

                    if let Some(next_value) = value

581

                        .checked_mul(max as UnsignedLarge)

582

                        .and_then(|value| value.checked_add(radix_value as UnsignedLarge))

583

584

                        value = next_value;

585

                    } else {

586

                        return Err(self.error(ErrorKind::IntegerTooLarge));

587

588

589

                Err('_') => {

590

                    self.chars.next();

591

592

                Err(_) => break,

593

594

595

596

        Ok(Token::new(

597

            self.chars.marked_range(),

598

            TokenKind::Integer(match (signed, negative) {

599

                (_, true) => Integer::SignedLarge(-(value as SignedLarge)),

600

                (true, _) => Integer::SignedLarge(value as SignedLarge),

601

                (false, _) => Integer::UnsignedLarge(value),

602

}),

603

))

604

605

606

    #[allow(clippy::cast_possible_wrap)]

607

43

    fn tokenize_radix_number<const RADIX: u32>(

608

43

        &mut self,

609

43

        signed: bool,

610

43

        negative: bool,

611

43

    ) -> Result<Token<'a>, Error> {

612

43

        assert!(RADIX == 1 || RADIX == 3 || RADIX == 4);

613

43

        let max = 2_u32.pow(RADIX);

614

43

        let mut value = 0usize;

615

43

        let mut read_at_least_one_digit = false;

616

617

738

        while let Some(result) = self.chars.peek().map(|ch| ch.to_digit(max).ok_or(ch)) {

618

701

            match result {

619

572

                Ok(radix_value) => {

620

572

                    read_at_least_one_digit = true;

621

572

                    self.chars.next();

622

572

                    if let Some(next_value) = value

623

572

                        .checked_mul(max as usize)

624

572

                        .and_then(|value| value.checked_add(radix_value as usize))

625

572

626

572

                        value = next_value;

627

572

                    } else {

628

                        // Overflowed

629

                        return self.tokenize_radix_large_number::<RADIX>(

630

                            signed,

631

                            negative,

632

                            value,

633

                            radix_value,

634

);

635

636

637

123

                Err('_') => {

638

123

                    self.chars.next();

639

123

640

6

                Err(_) => break,

641

642

643

644

43

        if read_at_least_one_digit {

645

            Ok(Token::new(

646

43

                self.chars.marked_range(),

647

43

                TokenKind::Integer(match (signed, negative) {

648

9

                    (_, true) => Integer::Isize(-(value as isize)),

649

9

                    (true, _) => Integer::Isize(value as isize),

650

25

                    (false, _) => Integer::Usize(value),

651

}),

652

))

653

        } else {

654

            Err(self.error(ErrorKind::ExpectedDigit))

655

656

43

657

658

125

    fn tokenize_leading_zero_number(

659

125

        &mut self,

660

125

        signed: bool,

661

125

        negative: bool,

662

125

    ) -> Result<Token<'a>, Error> {

663

125

        match self.chars.peek() {

664

            Some('x' | 'X') => {

665

15

                self.chars.next();

666

15

                return self.tokenize_radix_number::<4>(signed, negative);

667

668

            Some('b' | 'B') => {

669

14

                self.chars.next();

670

14

                return self.tokenize_radix_number::<1>(signed, negative);

671

672

            Some('o' | 'O') => {

673

14

                self.chars.next();

674

14

                return self.tokenize_radix_number::<3>(signed, negative);

675

676

82

            _ => {}

677

82

678

82

679

82

        match (signed, negative) {

680

1

            (_, true) => self.tokenize_negative_integer(0isize),

681

1

            (true, _) => self.tokenize_positive_integer(0isize),

682

80

            (false, _) => self.tokenize_positive_integer(0usize),

683

684

125

685

686

    #[allow(clippy::cast_possible_wrap)]

687

310

    fn tokenize_number(&mut self, start_char: u8) -> Result<Token<'a>, Error> {

688

310

        let negative = start_char == b'-';

689

310

        let signed = negative || start_char == b'+';

690

        // Check for inf/NaN

691

310

        if signed && matches!(self.chars.peek(), Some('i' | 'N')) {

692

            // We pass in 'i', but it doesn't matter as long as we provide a

693

            // xid_start character -- identifiers are always borrowed, so the

694

            // char is never passed to the output.

695

4

            return self.tokenize_identifier('i', false);

696

306

697

306

698

306

        if signed {

699

73

            let next_char = self.next_or_eof()?;

700

73

            if next_char == '0' {

701

20

                self.tokenize_leading_zero_number(signed, negative)

702

53

            } else if let Some(value) = next_char.to_digit(10) {

703

53

                let value = value as isize;

704

53

                if negative {

705

43

                    self.tokenize_negative_integer(-value)

706

                } else {

707

10

                    self.tokenize_positive_integer(value)

708

709

            } else {

710

                Err(self.error(ErrorKind::ExpectedDigit))

711

712

233

        } else if start_char == b'0' {

713

105

            self.tokenize_leading_zero_number(signed, negative)

714

        } else {

715

128

            let value = (start_char - b'0') as usize;

716

128

            self.tokenize_positive_integer(value)

717

718

310

719

720

24

    fn tokenize_char(&mut self) -> Result<Token<'a>, Error> {

721

24

        let ch = match self.next_or_eof()? {

722

18

            '\\' => self

723

18

                .tokenize_escaped_char::<false, true>()?

724

18

                .expect("underscore disallowed"),

725

            ch @ ('\n' | '\r' | '\t') => {

726

                return Err(self.error_at_last_char(ErrorKind::Unexpected(ch)))

727

728

6

            ch => ch,

729

};

730

731

        // Handle the trailing quote

732

24

        match self.next_or_eof()? {

733

24

            '\'' => Ok(Token::new(

734

24

                self.chars.marked_range(),

735

24

                TokenKind::Character(ch),

736

24

)),

737

            other => Err(self.error_at_last_char(ErrorKind::Unexpected(other))),

738

739

24

740

741

11

    fn tokenize_byte(&mut self) -> Result<Token<'a>, Error> {

742

11

        let ch = match self.next_or_eof()? {

743

9

            '\\' => self

744

9

                .tokenize_escaped_char::<false, false>()?

745

9

                .expect("underscore disallowed"),

746

2

            ch if ch.is_ascii() && !matches!(ch, '\n' | '\r' | '\t') => ch,

747

            ch => return Err(self.error_at_last_char(ErrorKind::Unexpected(ch))),

748

        } as u8;

749

750

        // Handle the trailing quote

751

11

        match self.next_or_eof()? {

752

11

            '\'' => Ok(Token::new(self.chars.marked_range(), TokenKind::Byte(ch))),

753

            other => Err(self.error_at_last_char(ErrorKind::Unexpected(other))),

754

755

11

756

757

36

    fn tokenize_string(&mut self) -> Result<Token<'a>, Error> {

758

        loop {

759

251

            match self.next_or_eof()? {

760

                '"' => {

761

                    // This string had no escapes, we can borrow.

762

29

                    let range = self.chars.marked_range();

763

29

                    let contents = &self.chars.source[range.start + 1..range.end - 1];

764

29

                    return Ok(Token::new(

765

29

                        range,

766

29

                        TokenKind::String(Cow::Borrowed(contents)),

767

29

));

768

769

7

                '\\' => break,

770

                '\r' => {

771

                    self.forbid_isolated_cr()?;

772

773

215

                _ => {}

774

775

776

777

        // We've encountered an escape sequence, which requires us to use our

778

        // scratch buffer to decode the escape sequences. First, we need to copy

779

        // everything over that wasn't escaped.

780

7

        self.scratch.clear();

781

7

        let starting_range = self.chars.marked_range();

782

7

        self.scratch

783

7

            .push_str(&self.chars.source[starting_range.start + 1..starting_range.end - 1]);

784

785

        loop {

786

            // Handle the escape sequence

787

48

            if let Some(ch) = self.tokenize_escaped_char::<true, true>()? {

788

47

                self.scratch.push(ch);

789

47

790

            // and then we resume a loop looking for the next escape sequence.

791

            loop {

792

143

                match self.next_or_eof()? {

793

                    '"' => {

794

7

                        return Ok(Token::new(

795

7

                            self.chars.marked_range(),

796

7

                            TokenKind::String(Cow::Owned(self.scratch.clone())),

797

7

));

798

799

41

                    '\\' => break,

800

                    '\r' => {

801

                        self.forbid_isolated_cr()?;

802

                        self.scratch.push('\r');

803

804

95

                    ch => {

805

95

                        self.scratch.push(ch);

806

95

807

808

809

810

36

811

812

75

    fn tokenize_escaped_char<const ALLOW_CONTINUE: bool, const ALLOW_UNICODE: bool>(

813

75

        &mut self,

814

75

    ) -> Result<Option<char>, Error> {

815

75

        // Now we need to handle the current escape sequnce

816

75

        match self.next_or_eof()? {

817

14

            ch @ ('"' | '\'' | '\\') => Ok(Some(ch)),

818

4

            'r' => Ok(Some('\r')),

819

2

            'n' => Ok(Some('\n')),

820

4

            't' => Ok(Some('\t')),

821

16

            '0' => Ok(Some('\0')),

822

2

            'u' if ALLOW_UNICODE => self.tokenize_unicode_escape().map(Some),

823

            'x' => {

824

32

                let escape_start = self.chars.last_offset();

825

32

                match self.tokenize_ascii_escape()? {

826

32

                    byte if byte.is_ascii() => Ok(Some(byte as char)),

827

                    _ => Err(Error::new(

828

                        escape_start..self.chars.last_offset(),

829

                        ErrorKind::InvalidAscii,

830

)),

831

832

833

            '\r' if ALLOW_CONTINUE => {

834

                self.forbid_isolated_cr()?;

835

                self.eat_whitespace_for_string_continue();

836

                Ok(None)

837

838

1

            '\n' if ALLOW_CONTINUE => {

839

1

                self.eat_whitespace_for_string_continue();

840

1

                Ok(None)

841

842

            ch => Err(self.error_at_last_char(ErrorKind::Unexpected(ch))),

843

844

75

845

846

2

    fn tokenize_unicode_escape(&mut self) -> Result<char, Error> {

847

2

        let start = self.chars.last_offset();

848

2

        // Open brace

849

2

        match self.next_or_eof()? {

850

2

            '{' => {}

851

            other => return Err(self.error_at_last_char(ErrorKind::Unexpected(other))),

852

853

854

2

        let mut possible_char = 0u32;

855

        loop {

856

14

            match self.next_or_eof()? {

857

                '}' => {

858

2

                    break;

859

860

2

                '_' => continue,

861

10

                ch => {

862

10

                    let radix_value = ch

863

10

                        .to_digit(16)

864

10

                        .ok_or_else(|| self.error_at_last_char(ErrorKind::InvalidUnicode))?;

865

866

10

                    if let Some(next_value) = possible_char.checked_shl(4) {

867

10

                        possible_char = next_value | radix_value;

868

10

                    } else {

869

                        // Overflowed

870

                        return Err(Error::new(

871

                            start..self.chars.last_offset(),

872

                            ErrorKind::InvalidUnicode,

873

));

874

875

876

877

878

879

2

        char::from_u32(possible_char)

880

2

            .ok_or_else(|| Error::new(start..self.chars.last_offset(), ErrorKind::InvalidUnicode))

881

2

882

883

    #[allow(clippy::cast_possible_truncation)]

884

189

    fn tokenize_ascii_escape(&mut self) -> Result<u8, Error> {

885

189

        let first_digit =

886

189

            self.next_or_eof()?

887

189

                .to_digit(16)

888

189

                .ok_or_else(|| self.error_at_last_char(ErrorKind::InvalidAscii))? as u8;

889

890

189

        let second_digit =

891

189

            self.next_or_eof()?

892

189

                .to_digit(16)

893

189

                .ok_or_else(|| self.error_at_last_char(ErrorKind::InvalidAscii))? as u8;

894

189

        Ok((first_digit << 4) | second_digit)

895

189

896

897

24

    fn tokenize_byte_string(&mut self) -> Result<Token<'a>, Error> {

898

        loop {

899

110

            match self.next_or_eof()? {

900

                '"' => {

901

11

                    let range = self.chars.marked_range();

902

11

                    let contents = &self.chars.source[range.start + 2..range.end - 1];

903

11

                    return Ok(Token::new(

904

11

                        range,

905

11

                        TokenKind::Bytes(Cow::Borrowed(contents.as_bytes())),

906

11

));

907

908

13

                '\\' => break,

909

86

                ch if ch.is_ascii() => {}

910

                _ => return Err(self.error_at_last_char(ErrorKind::InvalidAscii)),

911

912

913

914

        // We need a scratch buffer to handle escape sequences

915

13

        let mut scratch = mem::take(&mut self.scratch).into_bytes();

916

13

        scratch.clear();

917

13

        let start_range = self.chars.marked_range();

918

13

        scratch.extend_from_slice(

919

13

            self.chars.source[start_range.start + 2..start_range.end - 1].as_bytes(),

920

13

);

921

922

        loop {

923

174

            let byte = match self.next_or_eof()? {

924

5

                ch @ ('"' | '\'' | '\\') => Some(ch as u8),

925

2

                'r' => Some(b'\r'),

926

1

                'n' => Some(b'\n'),

927

2

                't' => Some(b'\t'),

928

6

                '0' => Some(b'\0'),

929

157

                'x' => Some(self.tokenize_ascii_escape()?),

930

                '\r' => {

931

                    self.forbid_isolated_cr()?;

932

                    self.eat_whitespace_for_string_continue();

933

                    None

934

935

                '\n' => {

936

1

                    self.eat_whitespace_for_string_continue();

937

1

                    None

938

939

                ch => return Err(self.error_at_last_char(ErrorKind::Unexpected(ch))),

940

};

941

942

174

            if let Some(byte) = byte {

943

173

                scratch.push(byte);

944

173

945

946

            loop {

947

268

                match self.next_or_eof()? {

948

                    '"' => {

949

                        // We want to keep the scratch buffer around to keep

950

                        // from needing to constantly grow it while parsing.

951

13

                        let contents = scratch.clone();

952

13

                        scratch.clear();

953

13

                        self.scratch = String::from_utf8(scratch).expect("empty vec");

954

13

                        return Ok(Token::new(

955

13

                            self.chars.marked_range(),

956

13

                            TokenKind::Bytes(Cow::Owned(contents)),

957

13

));

958

959

161

                    '\\' => break,

960

94

                    ch if ch.is_ascii() => {

961

94

                        scratch.push(ch as u8);

962

94

963

                    _ => return Err(self.error_at_last_char(ErrorKind::InvalidAscii)),

964

965

966

967

24

968

969

10

    fn tokenize_raw(&mut self) -> Result<Token<'a>, Error> {

970

10

        let mut pound_count = 0;

971

        // Count the number of leading pound signs

972

        loop {

973

21

            match self.next_or_eof()? {

974

11

                '#' => pound_count += 1,

975

6

                '"' => break,

976

4

                ch if pound_count == 1 && is_xid_start(ch) => {

977

4

                    return self.tokenize_identifier(ch, true);

978

979

                other => return Err(self.error_at_last_char(ErrorKind::Unexpected(other))),

980

981

982

983

        // String contents

984

        'contents: loop {

985

87

            match self.next_or_eof()? {

986

                '\r' => {

987

                    self.forbid_isolated_cr()?;

988

989

                '"' => {

990

14

                    let mut pounds_needed = pound_count;

991

23

                    while self.chars.peek() == Some('#') && pounds_needed > 0 {

992

9

                        self.chars.next();

993

9

                        pounds_needed -= 1;

994

9

995

996

                    // Only break if the correct number of pound signs has been

997

                    // encountered.

998

14

                    if pounds_needed == 0 {

999

6

                        break 'contents;

1000

8

1001

1002

73

                _ => {}

1003

1004

1005

1006

6

        let source_range = self.chars.marked_range();

1007

6

        let value = &self.chars.source

1008

6

            [source_range.start + pound_count + 2..source_range.end - pound_count - 1];

1009

6

        Ok(Token::new(

1010

6

            source_range,

1011

6

            TokenKind::String(Cow::Borrowed(value)),

1012

6

))

1013

10

1014

1015

6

    fn tokenize_raw_byte_string(&mut self) -> Result<Token<'a>, Error> {

1016

6

        // Count the number of leading pound signs

1017

6

        let mut pound_count = 0;

1018

        loop {

1019

13

            match self.next_or_eof()? {

1020

7

                '#' => pound_count += 1,

1021

6

                '"' => break,

1022

                other if pound_count == 0 => return self.tokenize_identifier(other, true),

1023

                other => return Err(self.error_at_last_char(ErrorKind::Unexpected(other))),

1024

1025

1026

1027

        // String contents

1028

        'contents: loop {

1029

87

            match self.next_or_eof()? {

1030

                '\r' => {

1031

                    self.forbid_isolated_cr()?;

1032

1033

                '"' => {

1034

14

                    let mut pounds_needed = pound_count;

1035

23

                    while self.chars.peek() == Some('#') && pounds_needed > 0 {

1036

9

                        self.chars.next();

1037

9

                        pounds_needed -= 1;

1038

9

1039

1040

                    // Only break if the correct number of pound signs has been

1041

                    // encountered.

1042

14

                    if pounds_needed == 0 {

1043

6

                        break 'contents;

1044

8

1045

1046

73

                ch if ch.is_ascii() => {}

1047

                _ => return Err(self.error_at_last_char(ErrorKind::InvalidAscii)),

1048

1049

1050

1051

6

        let source_range = self.chars.marked_range();

1052

6

        let value = &self.chars.source

1053

6

            [source_range.start + pound_count + 3..source_range.end - pound_count - 1];

1054

6

        Ok(Token::new(

1055

6

            source_range,

1056

6

            TokenKind::Bytes(Cow::Borrowed(value.as_bytes())),

1057

6

))

1058

6

1059

1060

    fn forbid_isolated_cr(&mut self) -> Result<(), Error> {

1061

        if self.chars.peek() == Some('\n') {

1062

            Ok(())

1063

        } else {

1064

            Err(self.error_at_last_char(ErrorKind::InvalidAscii))

1065

1066

1067

1068

2

    fn eat_whitespace_for_string_continue(&mut self) {

1069

18

        while self

1070

18

            .chars

1071

18

            .peek()

1072

18

            .map_or(false, |ch| matches!(ch, ' ' | '\n' | '\r' | '\t'))

1073

16

1074

16

            self.chars.next();

1075

16

1076

2

1077

1078

352

    fn tokenize_identifier(

1079

352

        &mut self,

1080

352

        initial_char: char,

1081

352

        is_raw: bool,

1082

352

    ) -> Result<Token<'a>, Error> {

1083

352

        let require_start = initial_char != '_';

1084

352

        let initial_char_index = self.chars.last_offset();

1085

        // Validate the first character

1086

352

        let start_is_valid = if require_start {

1087

350

            is_xid_start(initial_char)

1088

        } else {

1089

2

            is_xid_continue(initial_char)

1090

};

1091

1092

352

        if start_is_valid {

1093

1607

            while let Some(ch) = self.chars.peek() {

1094

1590

                if is_xid_continue(ch) {

1095

1256

                    self.chars.next();

1096

1256

                } else {

1097

334

                    break;

1098

1099

1100

1101

351

            let source = &self.chars.source[initial_char_index..self.chars.current_offset()];

1102

351

1103

351

            Ok(Token::new(

1104

351

                self.chars.marked_range(),

1105

1

                match source {

1106

351

                    "true" if !is_raw => TokenKind::Bool(true),

1107

338

                    "false" if !is_raw => TokenKind::Bool(false),

1108

323

                    "inf" | "+inf" if !is_raw => TokenKind::Float(f64::INFINITY),

1109

321

                    "NaN" | "+NaN" if !is_raw => TokenKind::Float(f64::NAN),

1110

319

                    "-inf" if !is_raw => TokenKind::Float(-f64::INFINITY),

1111

318

                    "-NaN" if !is_raw => TokenKind::Float(-f64::NAN),

1112

319

                    _ => TokenKind::Identifier(source),

1113

},

1114

))

1115

        } else {

1116

1

            Err(Error {

1117

1

                location: initial_char_index..self.chars.current_offset(),

1118

1

                kind: ErrorKind::Unexpected(initial_char),

1119

1

})

1120

1121

352

1122

1123

18

    fn tokenize_comment(&mut self) -> Result<Token<'a>, Error> {

1124

18

        match self.next_or_eof()? {

1125

16

            '*' => self.tokenize_block_comment(),

1126

2

            '/' => self.tokenize_single_line_comment(),

1127

            other => Err(self.error_at_last_char(ErrorKind::Unexpected(other))),

1128

1129

18

1130

1131

16

    fn tokenize_block_comment(&mut self) -> Result<Token<'a>, Error> {

1132

16

        let mut nests = 1;

1133

37

        while nests > 0 {

1134

37

            match self.next_or_eof()? {

1135

                '*' => {

1136

20

                    if self.chars.peek() == Some('/') {

1137

17

                        self.chars.next();

1138

17

                        nests -= 1;

1139

17

                        if nests == 0 {

1140

16

                            break;

1141

1

1142

3

1143

1144

                '/' => {

1145

1

                    if self.chars.peek() == Some('*') {

1146

1

                        self.chars.next();

1147

1

                        nests += 1;

1148

1

1149

1150

                '\r' => {

1151

                    self.forbid_isolated_cr()?;

1152

1153

16

                _ => {}

1154

1155

1156

1157

16

        Ok(Token::new(

1158

16

            self.chars.marked_range(),

1159

16

            TokenKind::Comment(&self.chars.source[self.chars.marked_range()]),

1160

16

))

1161

16

1162

1163

2

    fn tokenize_single_line_comment(&mut self) -> Result<Token<'a>, Error> {

1164

        loop {

1165

12

            match self.chars.peek() {

1166

                Some('\r') => {

1167

                    self.forbid_isolated_cr()?;

1168

                    break;

1169

1170

2

                Some('\n') | None => break,

1171

10

                _ => {

1172

10

                    self.chars.next();

1173

10

1174

1175

1176

2

        let range = self.chars.marked_range();

1177

2

        Ok(Token::new(

1178

2

            range.clone(),

1179

2

            TokenKind::Comment(&self.chars.source[range]),

1180

2

))

1181

2

1182

1183

1184

impl<'a, const INCLUDE_ALL: bool> Iterator for Tokenizer<'a, INCLUDE_ALL> {

1185

    type Item = Result<Token<'a>, Error>;

1186

1187

1645

    fn next(&mut self) -> Option<Self::Item> {

1188

        loop {

1189

2042

            self.chars.mark_start();

1190

2042

            let ch = self.chars.next()?;

1191

1434

            let result = match ch {

1192

1009

                '0'..='9' | '-' | '+' => self.tokenize_number(ch as u8),

1193

36

                '"' => self.tokenize_string(),

1194

24

                '\'' => self.tokenize_char(),

1195

14

                'r' => match self.chars.peek() {

1196

10

                    Some('#' | '"') => self.tokenize_raw(),

1197

4

                    _ => self.tokenize_identifier(ch, false),

1198

},

1199

86

                'b' => match self.chars.peek() {

1200

                    Some('r') => {

1201

6

                        self.chars.next();

1202

6

                        self.tokenize_raw_byte_string()

1203

1204

                    Some('"') => {

1205

24

                        self.chars.next();

1206

24

                        self.tokenize_byte_string()

1207

1208

                    Some('\'') => {

1209

11

                        self.chars.next();

1210

11

                        self.tokenize_byte()

1211

1212

45

                    _ => self.tokenize_identifier(ch, false),

1213

},

1214

23

                '(' => Ok(Token::new(

1215

23

                    self.chars.marked_range(),

1216

23

                    TokenKind::Open(Balanced::Paren),

1217

23

)),

1218

23

                ')' => Ok(Token::new(

1219

23

                    self.chars.marked_range(),

1220

23

                    TokenKind::Close(Balanced::Paren),

1221

23

)),

1222

34

                '{' => Ok(Token::new(

1223

34

                    self.chars.marked_range(),

1224

34

                    TokenKind::Open(Balanced::Brace),

1225

34

)),

1226

34

                '}' => Ok(Token::new(

1227

34

                    self.chars.marked_range(),

1228

34

                    TokenKind::Close(Balanced::Brace),

1229

34

)),

1230

19

                '[' => Ok(Token::new(

1231

19

                    self.chars.marked_range(),

1232

19

                    TokenKind::Open(Balanced::Bracket),

1233

19

)),

1234

18

                ']' => Ok(Token::new(

1235

18

                    self.chars.marked_range(),

1236

18

                    TokenKind::Close(Balanced::Bracket),

1237

18

)),

1238

276

                ':' => Ok(Token::new(self.chars.marked_range(), TokenKind::Colon)),

1239

245

                ',' => Ok(Token::new(self.chars.marked_range(), TokenKind::Comma)),

1240

711

                ch if is_rust_whitespace(ch) => {

1241

                    loop {

1242

762

                        match self.chars.peek() {

1243

757

                            Some(ch) if is_rust_whitespace(ch) => {

1244

364

                                self.chars.next();

1245

364

1246

398

                            _ => break,

1247

398

1248

398

1249

398

                    if INCLUDE_ALL {

1250

1

                        Ok(Token::new(

1251

1

                            self.chars.marked_range(),

1252

1

                            TokenKind::Whitespace(self.chars.marked_str()),

1253

1

))

1254

                    } else {

1255

397

                        continue;

1256

1257

1258

18

                '/' => self.tokenize_comment(),

1259

295

                ch => self.tokenize_identifier(ch, false),

1260

};

1261

1434

            break Some(result);

1262

1263

1623

1264

1265

1266

1468

fn is_rust_whitespace(ch: char) -> bool {

1267

    // https://doc.rust-lang.org/reference/whitespace.html

1268

706

    matches!(

1269

1468

ch,

1270

        '\t' | '\n'

1271

            | '\u{b}'

1272

            | '\u{c}'

1273

            | '\r'

1274

            | ' '

1275

            | '\u{85}'

1276

            | '\u{200e}'

1277

            | '\u{200f}'

1278

            | '\u{2028}'

1279

            | '\u{2029}'

1280

1281

1468

1282

1283

/// An error returned from a tokenizer.

1284

#[derive(Debug, Clone, Eq, PartialEq)]

1285

pub struct Error {

1286

    /// The byte range of the error.

1287

    pub location: Range<usize>,

1288

    /// The kind of error that occurred.

1289

    pub kind: ErrorKind,

1290

1291

1292

impl Error {

1293

    #[must_use]

1294

    const fn new(location: Range<usize>, kind: ErrorKind) -> Self {

1295

        Self { location, kind }

1296

1297

1298

1299

impl Display for Error {

1300

    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {

1301

        write!(f, "{} at {:?}", self.kind, self.location)

1302

1303

1304

1305

#[cfg(feature = "std")]

1306

impl std::error::Error for Error {}

1307

1308

/// The kind of an error returned from a tokenizer.

1309

#[derive(Debug, Clone, Eq, PartialEq)]

1310

#[non_exhaustive]

1311

pub enum ErrorKind {

1312

    /// The end of input was encountered when additional data was still expected.

1313

    UnexpectedEof,

1314

    /// An unexpected character was found.

1315

    Unexpected(char),

1316

    /// A digit (0-9) was expected.

1317

    ExpectedDigit,

1318

    /// The integer literal is too large to represent.

1319

    IntegerTooLarge,

1320

    /// An invalid unicode code point was encountered.

1321

    InvalidUnicode,

1322

    /// An invalid ASCII character was encountered.

1323

    InvalidAscii,

1324

    /// An invalid floating point number was found.

1325

    InvalidFloat,

1326

    /// A carriage return without a corresponding `\n` was encountered.

1327

    IsolatedCarriageReturn,

1328

1329

1330

impl Display for ErrorKind {

1331

    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {

1332

        match self {

1333

            ErrorKind::UnexpectedEof => f.write_str("unexpected eof"),

1334

            ErrorKind::Unexpected(ch) => write!(f, "unexpected `{ch}`"),

1335

            ErrorKind::ExpectedDigit => f.write_str("expected digit"),

1336

            ErrorKind::InvalidUnicode => f.write_str("invalid unicode escape sequence"),

1337

            ErrorKind::InvalidAscii => f.write_str("invalid ascii escape sequence"),

1338

            ErrorKind::IsolatedCarriageReturn => f.write_str("unexpected isolated carriage return"),

1339

            ErrorKind::IntegerTooLarge => f.write_str("value overflowed the maximum size"),

1340

            ErrorKind::InvalidFloat => f.write_str("invalid floating point literal"),

1341

1342

1343

1344

1345

trait Integral: From<u8> + Copy {

1346

    type Larger: Integral;

1347

    fn into_larger(self) -> Self::Larger;

1348

    fn checked_mul(self, other: Self) -> Option<Self>;

1349

    fn checked_add(self, other: Self) -> Option<Self>;

1350

    fn checked_sub(self, other: Self) -> Option<Self>;

1351

1352

1353

impl Integral for usize {

1354

    type Larger = UnsignedLarge;

1355

1356

    fn into_larger(self) -> Self::Larger {

1357

        self as UnsignedLarge

1358

1359

1360

606

    fn checked_mul(self, other: Self) -> Option<Self> {

1361

606

        self.checked_mul(other)

1362

606

1363

1364

606

    fn checked_add(self, other: Self) -> Option<Self> {

1365

606

        self.checked_add(other)

1366

606

1367

1368

    fn checked_sub(self, other: Self) -> Option<Self> {

1369

        self.checked_sub(other)

1370

1371

1372

1373

impl Integral for UnsignedLarge {

1374

    type Larger = Self;

1375

1376

    fn into_larger(self) -> Self::Larger {

1377

        self

1378

1379

1380

    fn checked_mul(self, other: Self) -> Option<Self> {

1381

        self.checked_mul(other)

1382

1383

1384

    fn checked_add(self, other: Self) -> Option<Self> {

1385

        self.checked_add(other)

1386

1387

1388

    fn checked_sub(self, other: Self) -> Option<Self> {

1389

        self.checked_sub(other)

1390

1391

1392

1393

impl Integral for isize {

1394

    type Larger = SignedLarge;

1395

1396

    fn into_larger(self) -> Self::Larger {

1397

        self as SignedLarge

1398

1399

1400

342

    fn checked_mul(self, other: Self) -> Option<Self> {

1401

342

        self.checked_mul(other)

1402

342

1403

1404

32

    fn checked_add(self, other: Self) -> Option<Self> {

1405

32

        self.checked_add(other)

1406

32

1407

1408

310

    fn checked_sub(self, other: Self) -> Option<Self> {

1409

310

        self.checked_sub(other)

1410

310

1411

1412

1413

impl Integral for SignedLarge {

1414

    type Larger = Self;

1415

1416

    fn into_larger(self) -> Self::Larger {

1417

        self

1418

1419

1420

    fn checked_mul(self, other: Self) -> Option<Self> {

1421

        self.checked_mul(other)

1422

1423

1424

    fn checked_add(self, other: Self) -> Option<Self> {

1425

        self.checked_add(other)

1426

1427

1428

    fn checked_sub(self, other: Self) -> Option<Self> {

1429

        self.checked_sub(other)

1430

1431

1432

1433

#[cfg(test)]

1434

#[allow(clippy::too_many_lines)]

1435

mod tests {

1436

    use alloc::vec::Vec;

1437

1438

    use super::*;

1439

1440

    #[track_caller]

1441

119

    fn test_tokens(source: &str, tokens: &[Token<'_>]) {

1442

119

        assert_eq!(

1443

119

            &Tokenizer::minified(source)

1444

119

                .collect::<Result<Vec<_>, _>>()

1445

119

                .unwrap(),

1446

119

            tokens

1447

119

);

1448

119

1449

1450

    #[track_caller]

1451

4

    fn test_tokens_full(source: &str, tokens: &[Token<'_>]) {

1452

4

        assert_eq!(

1453

4

            &Tokenizer::full(source)

1454

4

                .collect::<Result<Vec<_>, _>>()

1455

4

                .unwrap(),

1456

4

            tokens

1457

4

);

1458

4

1459

1460

    #[track_caller]

1461

1

    fn test_tokens_err(source: &str, location: Range<usize>, kind: &ErrorKind) {

1462

1

        let err = Tokenizer::minified(source)

1463

1

            .collect::<Result<Vec<_>, _>>()

1464

1

            .expect_err("source did not error");

1465

1

        assert_eq!(&err.kind, kind);

1466

1

        assert_eq!(err.location, location);

1467

1

1468

1469

    #[test]

1470

1

    fn identifiers() {

1471

1

        test_tokens("true", &[Token::new(0..4, TokenKind::Bool(true))]);

1472

1

        test_tokens("false", &[Token::new(0..5, TokenKind::Bool(false))]);

1473

1

1474

1

        test_tokens("r#true", &[Token::new(0..6, TokenKind::Identifier("true"))]);

1475

1

        test_tokens(

1476

1

            "r#false",

1477

1

            &[Token::new(0..7, TokenKind::Identifier("false"))],

1478

1

);

1479

1

1480

1

        test_tokens("_", &[Token::new(0..1, TokenKind::Identifier("_"))]);

1481

1

1482

1

        test_tokens("_0", &[Token::new(0..2, TokenKind::Identifier("_0"))]);

1483

1

1484

1

        test_tokens_err("=", 0..1, &ErrorKind::Unexpected('='));

1485

1

1486

1487

    #[test]

1488

1

    fn integers() {

1489

1

        test_tokens(

1490

1

            "0",

1491

1

            &[Token::new(0..1, TokenKind::Integer(Integer::Usize(0)))],

1492

1

);

1493

1

        test_tokens(

1494

1

            "9",

1495

1

            &[Token::new(0..1, TokenKind::Integer(Integer::Usize(9)))],

1496

1

);

1497

1

        test_tokens(

1498

1

            "10",

1499

1

            &[Token::new(0..2, TokenKind::Integer(Integer::Usize(10)))],

1500

1

);

1501

1

        test_tokens(

1502

1

            "99",

1503

1

            &[Token::new(0..2, TokenKind::Integer(Integer::Usize(99)))],

1504

1

);

1505

1

        test_tokens(

1506

1

            "+0",

1507

1

            &[Token::new(0..2, TokenKind::Integer(Integer::Isize(0)))],

1508

1

);

1509

1

        test_tokens(

1510

1

            "+9",

1511

1

            &[Token::new(0..2, TokenKind::Integer(Integer::Isize(9)))],

1512

1

);

1513

1

        test_tokens(

1514

1

            "-0",

1515

1

            &[Token::new(0..2, TokenKind::Integer(Integer::Isize(0)))],

1516

1

);

1517

1

        test_tokens(

1518

1

            "-9",

1519

1

            &[Token::new(0..2, TokenKind::Integer(Integer::Isize(-9)))],

1520

1

);

1521

1

        test_tokens(

1522

1

            "-10",

1523

1

            &[Token::new(0..3, TokenKind::Integer(Integer::Isize(-10)))],

1524

1

);

1525

1

        test_tokens(

1526

1

            "-99",

1527

1

            &[Token::new(0..3, TokenKind::Integer(Integer::Isize(-99)))],

1528

1

);

1529

1

1530

1

        // Test 16-bit integer maximums

1531

1

        test_tokens(

1532

1

            "+32_767",

1533

1

            &[Token::new(0..7, TokenKind::Integer(Integer::Isize(32_767)))],

1534

1

);

1535

1

        test_tokens(

1536

1

            "-32_768",

1537

1

            &[Token::new(

1538

1

                0..7,

1539

1

                TokenKind::Integer(Integer::Isize(-32_768)),

1540

1

)],

1541

1

);

1542

1

        test_tokens(

1543

1

            "65_535",

1544

1

            &[Token::new(0..6, TokenKind::Integer(Integer::Usize(65_535)))],

1545

1

);

1546

1

1547

1

        // Test 32-bit integer maximums

1548

1

        test_tokens(

1549

1

            "+2_147_483_647",

1550

1

            &[Token::new(

1551

1

                0..14,

1552

1

                #[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))]

1553

1

                TokenKind::Integer(Integer::Isize(2_147_483_647)),

1554

1

                #[cfg(target_pointer_width = "16")]

1555

1

                TokenKind::Integer(Integer::SignedLarge(2_147_483_647)),

1556

1

)],

1557

1

);

1558

1

        test_tokens(

1559

1

            "-2_147_483_648",

1560

1

            &[Token::new(

1561

1

                0..14,

1562

1

                #[cfg(target_pointer_width = "64")]

1563

1

                TokenKind::Integer(Integer::Isize(-2_147_483_648)),

1564

1

                #[cfg(not(target_pointer_width = "64"))]

1565

1

                TokenKind::Integer(Integer::SignedLarge(-2_147_483_648)),

1566

1

)],

1567

1

);

1568

1

        test_tokens(

1569

1

            "4_294_967_295",

1570

1

            &[Token::new(

1571

1

                0..13,

1572

1

                #[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))]

1573

1

                TokenKind::Integer(Integer::Usize(4_294_967_295)),

1574

1

                #[cfg(target_pointer_width = "16")]

1575

1

                TokenKind::Integer(Integer::UnsignedLarge(4_294_967_295)),

1576

1

)],

1577

1

);

1578

1

1579

1

        // Test 64-bit integer maximums

1580

1

        test_tokens(

1581

1

            "+9_223_372_036_854_775_807",

1582

1

            &[Token::new(

1583

1

                0..26,

1584

1

                #[cfg(target_pointer_width = "64")]

1585

1

                TokenKind::Integer(Integer::Isize(9_223_372_036_854_775_807)),

1586

1

                #[cfg(not(target_pointer_width = "64"))]

1587

1

                TokenKind::Integer(Integer::SignedLarge(9_223_372_036_854_775_807)),

1588

1

)],

1589

1

);

1590

1

        test_tokens(

1591

1

            "-9_223_372_036_854_775_808",

1592

1

            &[Token::new(

1593

1

                0..26,

1594

1

                #[cfg(target_pointer_width = "64")]

1595

1

                TokenKind::Integer(Integer::Isize(-9_223_372_036_854_775_808)),

1596

1

                #[cfg(not(target_pointer_width = "64"))]

1597

1

                TokenKind::Integer(Integer::SignedLarge(-9_223_372_036_854_775_808)),

1598

1

)],

1599

1

);

1600

1

        test_tokens(

1601

1

            "18_446_744_073_709_551_615",

1602

1

            &[Token::new(

1603

1

                0..26,

1604

1

                #[cfg(target_pointer_width = "64")]

1605

1

                TokenKind::Integer(Integer::Usize(18_446_744_073_709_551_615)),

1606

1

                #[cfg(not(target_pointer_width = "64"))]

1607

1

                TokenKind::Integer(Integer::UnsignedLarge(18_446_744_073_709_551_615)),

1608

1

)],

1609

1

);

1610

1

1611

1

        #[cfg(feature = "integer128")]

1612

1

1613

1

            test_tokens(

1614

1

                "+9_223_372_036_854_775_808",

1615

1

                &[Token::new(

1616

1

                    0..26,

1617

1

                    TokenKind::Integer(Integer::SignedLarge(9_223_372_036_854_775_808)),

1618

1

)],

1619

1

);

1620

1

            test_tokens(

1621

1

                "-9_223_372_036_854_775_809",

1622

1

                &[Token::new(

1623

1

                    0..26,

1624

1

                    TokenKind::Integer(Integer::SignedLarge(-9_223_372_036_854_775_809)),

1625

1

)],

1626

1

);

1627

1

            test_tokens(

1628

1

                "18_446_744_073_709_551_616",

1629

1

                &[Token::new(

1630

1

                    0..26,

1631

1

                    TokenKind::Integer(Integer::UnsignedLarge(18_446_744_073_709_551_616)),

1632

1

)],

1633

1

);

1634

1

1635

1

1636

1637

    #[test]

1638

1

    fn hex_integers() {

1639

1

        test_tokens(

1640

1

            "0x1",

1641

1

            &[Token::new(0..3, TokenKind::Integer(Integer::Usize(1)))],

1642

1

);

1643

1

        test_tokens(

1644

1

            "0X12",

1645

1

            &[Token::new(0..4, TokenKind::Integer(Integer::Usize(0x12)))],

1646

1

);

1647

1

        test_tokens(

1648

1

            "0x12_3",

1649

1

            &[Token::new(0..6, TokenKind::Integer(Integer::Usize(0x123)))],

1650

1

);

1651

1

        test_tokens(

1652

1

            "0xaBc",

1653

1

            &[Token::new(0..5, TokenKind::Integer(Integer::Usize(0xabc)))],

1654

1

);

1655

1

1656

1

        // Test 16-bit integer maximums

1657

1

        test_tokens(

1658

1

            "+0xFFFF",

1659

1

            &[Token::new(0..7, TokenKind::Integer(Integer::Isize(0xFFFF)))],

1660

1

);

1661

1

        test_tokens(

1662

1

            "-0xFFFF",

1663

1

            &[Token::new(

1664

1

                0..7,

1665

1

                TokenKind::Integer(Integer::Isize(-0xFFFF)),

1666

1

)],

1667

1

);

1668

1

        test_tokens(

1669

1

            "0xFFFF",

1670

1

            &[Token::new(0..6, TokenKind::Integer(Integer::Usize(0xFFFF)))],

1671

1

);

1672

1

1673

1

        // Test 32-bit integer maximums

1674

1

        test_tokens(

1675

1

            "+0xFFFF_FFFF",

1676

1

            &[Token::new(

1677

1

                0..12,

1678

1

                #[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))]

1679

1

                TokenKind::Integer(Integer::Isize(0xFFFF_FFFF)),

1680

1

                #[cfg(target_pointer_width = "16")]

1681

1

                TokenKind::Integer(Integer::SignedLarge(0xFFFF_FFFF)),

1682

1

)],

1683

1

);

1684

1

        test_tokens(

1685

1

            "-0xFFFF_FFFF",

1686

1

            &[Token::new(

1687

1

                0..12,

1688

1

                #[cfg(target_pointer_width = "64")]

1689

1

                TokenKind::Integer(Integer::Isize(-0xFFFF_FFFF)),

1690

1

                #[cfg(not(target_pointer_width = "64"))]

1691

1

                TokenKind::Integer(Integer::SignedLarge(-0xFFFF_FFFF)),

1692

1

)],

1693

1

);

1694

1

        test_tokens(

1695

1

            "0xFFFF_FFFF",

1696

1

            &[Token::new(

1697

1

                0..11,

1698

1

                #[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))]

1699

1

                TokenKind::Integer(Integer::Usize(0xFFFF_FFFF)),

1700

1

                #[cfg(target_pointer_width = "16")]

1701

1

                TokenKind::Integer(Integer::UnsignedLarge(0xFFFF_FFFF)),

1702

1

)],

1703

1

);

1704

1

1705

1

        // Test 64-bit integer maximums

1706

1

        #[allow(overflowing_literals)]

1707

1

        test_tokens(

1708

1

            "+0xFFFF_FFFF_FFFF_FFFF",

1709

1

            &[Token::new(

1710

1

                0..22,

1711

1

                #[cfg(target_pointer_width = "64")]

1712

1

                TokenKind::Integer(Integer::Isize(0xFFFF_FFFF_FFFF_FFFF)),

1713

1

                #[cfg(not(target_pointer_width = "64"))]

1714

1

                TokenKind::Integer(Integer::SignedLarge(0xFFFF_FFFF_FFFF_FFFF)),

1715

1

)],

1716

1

);

1717

1

        #[allow(overflowing_literals)]

1718

1

        test_tokens(

1719

1

            "-0xFFFF_FFFF_FFFF_FFFF",

1720

1

            &[Token::new(

1721

1

                0..22,

1722

1

                #[cfg(target_pointer_width = "64")]

1723

1

                TokenKind::Integer(Integer::Isize(-0xFFFF_FFFF_FFFF_FFFF)),

1724

1

                #[cfg(not(target_pointer_width = "64"))]

1725

1

                TokenKind::Integer(Integer::SignedLarge(-0xFFFF_FFFF_FFFF_FFFF)),

1726

1

)],

1727

1

);

1728

1

        test_tokens(

1729

1

            "0xFFFF_FFFF_FFFF_FFFF",

1730

1

            &[Token::new(

1731

1

                0..21,

1732

1

                #[cfg(target_pointer_width = "64")]

1733

1

                TokenKind::Integer(Integer::Usize(0xFFFF_FFFF_FFFF_FFFF)),

1734

1

                #[cfg(not(target_pointer_width = "64"))]

1735

1

                TokenKind::Integer(Integer::UnsignedLarge(0xFFFF_FFFF_FFFF_FFFF)),

1736

1

)],

1737

1

);

1738

1

1739

1

        #[cfg(feature = "integer128")]

1740

1

1741

1

            #[allow(overflowing_literals)]

1742

1

            test_tokens(

1743

1

                "+0xFFFF_FFFF_FFFF_FFFF_FFFF_FFFF_FFFF_FFFF",

1744

1

                &[Token::new(

1745

1

                    0..42,

1746

1

                    TokenKind::Integer(Integer::SignedLarge(

1747

1

                        0xFFFF_FFFF_FFFF_FFFF_FFFF_FFFF_FFFF_FFFF,

1748

1

)),

1749

1

)],

1750

1

);

1751

1

            #[allow(overflowing_literals)]

1752

1

            test_tokens(

1753

1

                "-0xFFFF_FFFF_FFFF_FFFF_FFFF_FFFF_FFFF_FFFF",

1754

1

                &[Token::new(

1755

1

                    0..42,

1756

1

                    TokenKind::Integer(Integer::SignedLarge(

1757

1

                        -0xFFFF_FFFF_FFFF_FFFF_FFFF_FFFF_FFFF_FFFF,

1758

1

)),

1759

1

)],

1760

1

);

1761

1

            test_tokens(

1762

1

                "0xFFFF_FFFF_FFFF_FFFF_FFFF_FFFF_FFFF_FFFF",

1763

1

                &[Token::new(

1764

1

                    0..41,

1765

1

                    TokenKind::Integer(Integer::UnsignedLarge(

1766

1

                        0xFFFF_FFFF_FFFF_FFFF_FFFF_FFFF_FFFF_FFFF,

1767

1

)),

1768

1

)],

1769

1

);

1770

1

1771

1

1772

1773

    #[test]

1774

    #[allow(overflowing_literals)]

1775

1

    fn octal_integers() {

1776

1

        test_tokens(

1777

1

            "0o1",

1778

1

            &[Token::new(0..3, TokenKind::Integer(Integer::Usize(1)))],

1779

1

);

1780

1

        test_tokens(

1781

1

            "0O12",

1782

1

            &[Token::new(0..4, TokenKind::Integer(Integer::Usize(0o12)))],

1783

1

);

1784

1

        test_tokens(

1785

1

            "0o12_3",

1786

1

            &[Token::new(0..6, TokenKind::Integer(Integer::Usize(0o123)))],

1787

1

);

1788

1

1789

1

        // Test 16-bit integer maximums

1790

1

        test_tokens(

1791

1

            "+0o177_777",

1792

1

            &[Token::new(

1793

1

                0..10,

1794

1

                TokenKind::Integer(Integer::Isize(0o177_777)),

1795

1

)],

1796

1

);

1797

1

        test_tokens(

1798

1

            "-0o177_777",

1799

1

            &[Token::new(

1800

1

                0..10,

1801

1

                TokenKind::Integer(Integer::Isize(-0o177_777)),

1802

1

)],

1803

1

);

1804

1

        test_tokens(

1805

1

            "0o177_777",

1806

1

            &[Token::new(

1807

1

                0..9,

1808

1

                TokenKind::Integer(Integer::Usize(0o177_777)),

1809

1

)],

1810

1

);

1811

1

1812

1

        // Test 32-bit integer maximums

1813

1

        test_tokens(

1814

1

            "+0o37_777_777_777",

1815

1

            &[Token::new(

1816

1

                0..17,

1817

1

                #[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))]

1818

1

                TokenKind::Integer(Integer::Isize(0o37_777_777_777)),

1819

1

                #[cfg(target_pointer_width = "16")]

1820

1

                TokenKind::Integer(Integer::SignedLarge(0o37_777_777_777)),

1821

1

)],

1822

1

);

1823

1

        test_tokens(

1824

1

            "-0o37_777_777_777",

1825

1

            &[Token::new(

1826

1

                0..17,

1827

1

                #[cfg(target_pointer_width = "64")]

1828

1

                TokenKind::Integer(Integer::Isize(-0o37_777_777_777)),

1829

1

                #[cfg(not(target_pointer_width = "64"))]

1830

1

                TokenKind::Integer(Integer::SignedLarge(-0o37_777_777_777)),

1831

1

)],

1832

1

);

1833

1

        test_tokens(

1834

1

            "0o37_777_777_777",

1835

1

            &[Token::new(

1836

1

                0..16,

1837

1

                #[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))]

1838

1

                TokenKind::Integer(Integer::Usize(0o37_777_777_777)),

1839

1

                #[cfg(target_pointer_width = "16")]

1840

1

                TokenKind::Integer(Integer::UnsignedLarge(0o37_777_777_777)),

1841

1

)],

1842

1

);

1843

1

1844

1

        // Test 64-bit integer maximums

1845

1

        test_tokens(

1846

1

            "+0o1_777_777_777_777_777_777_777",

1847

1

            &[Token::new(

1848

1

                0..32,

1849

1

                #[cfg(target_pointer_width = "64")]

1850

1

                TokenKind::Integer(Integer::Isize(0o1_777_777_777_777_777_777_777)),

1851

1

                #[cfg(not(target_pointer_width = "64"))]

1852

1

                TokenKind::Integer(Integer::SignedLarge(0o1_777_777_777_777_777_777_777)),

1853

1

)],

1854

1

);

1855

1

        test_tokens(

1856

1

            "-0o1_777_777_777_777_777_777_777",

1857

1

            &[Token::new(

1858

1

                0..32,

1859

1

                #[cfg(target_pointer_width = "64")]

1860

1

                TokenKind::Integer(Integer::Isize(-0o1_777_777_777_777_777_777_777)),

1861

1

                #[cfg(not(target_pointer_width = "64"))]

1862

1

                TokenKind::Integer(Integer::SignedLarge(-0o1_777_777_777_777_777_777_777)),

1863

1

)],

1864

1

);

1865

1

        test_tokens(

1866

1

            "0o1_777_777_777_777_777_777_777",

1867

1

            &[Token::new(

1868

1

                0..31,

1869

1

                #[cfg(target_pointer_width = "64")]

1870

1

                TokenKind::Integer(Integer::Usize(0o1_777_777_777_777_777_777_777)),

1871

1

                #[cfg(not(target_pointer_width = "64"))]

1872

1

                TokenKind::Integer(Integer::UnsignedLarge(0o1_777_777_777_777_777_777_777)),

1873

1

)],

1874

1

);

1875

1

1876

1

        #[cfg(feature = "integer128")]

1877

1

1878

1

            test_tokens(

1879

1

                "+0o3_777_777_777_777_777_777_777_777_777_777_777_777_777_777",

1880

1

                &[Token::new(

1881

1

                    0..60,

1882

1

                    TokenKind::Integer(Integer::SignedLarge(

1883

1

                        0o3_777_777_777_777_777_777_777_777_777_777_777_777_777_777,

1884

1

)),

1885

1

)],

1886

1

);

1887

1

            test_tokens(

1888

1

                "-0o3_777_777_777_777_777_777_777_777_777_777_777_777_777_777",

1889

1

                &[Token::new(

1890

1

                    0..60,

1891

1

                    TokenKind::Integer(Integer::SignedLarge(

1892

1

                        -0o3_777_777_777_777_777_777_777_777_777_777_777_777_777_777,

1893

1

)),

1894

1

)],

1895

1

);

1896

1

            test_tokens(

1897

1

                "0o3_777_777_777_777_777_777_777_777_777_777_777_777_777_777",

1898

1

                &[Token::new(

1899

1

                    0..59,

1900

1

                    TokenKind::Integer(Integer::UnsignedLarge(

1901

1

                        0o3_777_777_777_777_777_777_777_777_777_777_777_777_777_777,

1902

1

)),

1903

1

)],

1904

1

);

1905

1

1906

1

1907

1908

    #[test]

1909

    #[allow(overflowing_literals)]

1910

1

    fn binary_integers() {

1911

1

        test_tokens(

1912

1

            "0b1",

1913

1

            &[Token::new(0..3, TokenKind::Integer(Integer::Usize(1)))],

1914

1

);

1915

1

        test_tokens(

1916

1

            "0B10",

1917

1

            &[Token::new(0..4, TokenKind::Integer(Integer::Usize(0b10)))],

1918

1

);

1919

1

        test_tokens(

1920

1

            "0b10_1",

1921

1

            &[Token::new(0..6, TokenKind::Integer(Integer::Usize(0b101)))],

1922

1

);

1923

1

1924

1

        // Test 16-bit integer maximums

1925

1

        test_tokens(

1926

1

            "+0b1111_1111_1111_1111",

1927

1

            &[Token::new(

1928

1

                0..22,

1929

1

                TokenKind::Integer(Integer::Isize(0b1111_1111_1111_1111)),

1930

1

)],

1931

1

);

1932

1

        test_tokens(

1933

1

            "-0b1111_1111_1111_1111",

1934

1

            &[Token::new(

1935

1

                0..22,

1936

1

                TokenKind::Integer(Integer::Isize(-0b1111_1111_1111_1111)),

1937

1

)],

1938

1

);

1939

1

        test_tokens(

1940

1

            "0b1111_1111_1111_1111",

1941

1

            &[Token::new(

1942

1

                0..21,

1943

1

                TokenKind::Integer(Integer::Usize(0b1111_1111_1111_1111)),

1944

1

)],

1945

1

);

1946

1

1947

1

        // Test 32-bit integer maximums

1948

1

        test_tokens(

1949

1

            "+0b1111_1111_1111_1111_1111_1111_1111_1111",

1950

1

            &[Token::new(

1951

1

                0..42,

1952

1

                #[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))]

1953

1

                TokenKind::Integer(Integer::Isize(0b1111_1111_1111_1111_1111_1111_1111_1111)),

1954

1

                #[cfg(target_pointer_width = "16")]

1955

1

                TokenKind::Integer(Integer::SignedLarge(

1956

1

                    0b1111_1111_1111_1111_1111_1111_1111_1111,

1957

1

)),

1958

1

)],

1959

1

);

1960

1

        test_tokens(

1961

1

            "-0b1111_1111_1111_1111_1111_1111_1111_1111",

1962

1

            &[Token::new(

1963

1

                0..42,

1964

1

                #[cfg(target_pointer_width = "64")]

1965

1

                TokenKind::Integer(Integer::Isize(-0b1111_1111_1111_1111_1111_1111_1111_1111)),

1966

1

                #[cfg(not(target_pointer_width = "64"))]

1967

1

                TokenKind::Integer(Integer::SignedLarge(

1968

1

                    -0b1111_1111_1111_1111_1111_1111_1111_1111,

1969

1

)),

1970

1

)],

1971

1

);

1972

1

        test_tokens(

1973

1

            "0b1111_1111_1111_1111_1111_1111_1111_1111",

1974

1

            &[Token::new(

1975

1

                0..41,

1976

1

                #[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))]

1977

1

                TokenKind::Integer(Integer::Usize(0b1111_1111_1111_1111_1111_1111_1111_1111)),

1978

1

                #[cfg(target_pointer_width = "16")]

1979

1

                TokenKind::Integer(Integer::UnsignedLarge(

1980

1

                    0b1111_1111_1111_1111_1111_1111_1111_1111,

1981

1

)),

1982

1

)],

1983

1

);

1984

1

1985

1

        // Test 64-bit integer maximums

1986

1

        test_tokens(

1987

1

            "+0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111",

1988

1

            &[Token::new(

1989

1

                0..82,

1990

1

                #[cfg(target_pointer_width = "64")]

1991

1

                TokenKind::Integer(Integer::Isize(0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111)),

1992

1

                #[cfg(not(target_pointer_width = "64"))]

1993

1

                TokenKind::Integer(Integer::SignedLarge(0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111)),

1994

1

)],

1995

1

);

1996

1

        test_tokens(

1997

1

            "-0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111",

1998

1

            &[Token::new(

1999

1

                0..82,

2000

1

                #[cfg(target_pointer_width = "64")]

2001

1

                TokenKind::Integer(Integer::Isize(-0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111)),

2002

1

                #[cfg(not(target_pointer_width = "64"))]

2003

1

                TokenKind::Integer(Integer::SignedLarge(-0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111)),

2004

1

)],

2005

1

);

2006

1

        test_tokens(

2007

1

            "0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111",

2008

1

            &[Token::new(

2009

1

                0..81,

2010

1

                #[cfg(target_pointer_width = "64")]

2011

1

                TokenKind::Integer(Integer::Usize(0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111)),

2012

1

                #[cfg(not(target_pointer_width = "64"))]

2013

1

                TokenKind::Integer(Integer::UnsignedLarge(0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111)),

2014

1

)],

2015

1

);

2016

1

2017

1

        #[cfg(feature = "integer128")]

2018

1

2019

1

            test_tokens(

2020

1

                "+0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111",

2021

1

                &[Token::new(

2022

1

                    0..162,

2023

1

                    TokenKind::Integer(Integer::SignedLarge(

2024

1

                        0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111,

2025

1

)),

2026

1

)],

2027

1

);

2028

1

            test_tokens(

2029

1

                "-0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111",

2030

1

                &[Token::new(

2031

1

                    0..162,

2032

1

                    TokenKind::Integer(Integer::SignedLarge(

2033

1

                        -0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111,

2034

1

)),

2035

1

)],

2036

1

);

2037

1

            test_tokens(

2038

1

                "0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111",

2039

1

                &[Token::new(

2040

1

                    0..161,

2041

1

                    TokenKind::Integer(Integer::UnsignedLarge(

2042

1

                        0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111,

2043

1

)),

2044

1

)],

2045

1

);

2046

1

2047

1

2048

2049

    #[test]

2050

1

    fn floats() {

2051

1

        test_tokens("0.", &[Token::new(0..2, TokenKind::Float(0.))]);

2052

1

        test_tokens("1.0", &[Token::new(0..3, TokenKind::Float(1.))]);

2053

1

        test_tokens("-1.0", &[Token::new(0..4, TokenKind::Float(-1.))]);

2054

1

        test_tokens("+1.0", &[Token::new(0..4, TokenKind::Float(1.))]);

2055

1

        test_tokens("-1.0e1", &[Token::new(0..6, TokenKind::Float(-10.))]);

2056

1

        test_tokens("+1.0e1", &[Token::new(0..6, TokenKind::Float(10.))]);

2057

1

        test_tokens("-1.0e+1", &[Token::new(0..7, TokenKind::Float(-10.))]);

2058

1

        test_tokens("+1.0e+1", &[Token::new(0..7, TokenKind::Float(10.))]);

2059

1

        test_tokens("-10.0e-1", &[Token::new(0..8, TokenKind::Float(-1.))]);

2060

1

        test_tokens("+10.0e-1", &[Token::new(0..8, TokenKind::Float(1.))]);

2061

1

        test_tokens("-1.0e10", &[Token::new(0..7, TokenKind::Float(-1e10))]);

2062

1

        test_tokens("+1.0e10", &[Token::new(0..7, TokenKind::Float(1.0e10))]);

2063

1

        test_tokens("-1e10", &[Token::new(0..5, TokenKind::Float(-1e10))]);

2064

1

        test_tokens("+1e10", &[Token::new(0..5, TokenKind::Float(1e10))]);

2065

1

        test_tokens("inf", &[Token::new(0..3, TokenKind::Float(f64::INFINITY))]);

2066

1

        test_tokens("NaN", &[Token::new(0..3, TokenKind::Float(f64::NAN))]);

2067

1

        test_tokens(

2068

1

            "-inf",

2069

1

            &[Token::new(0..4, TokenKind::Float(-f64::INFINITY))],

2070

1

);

2071

1

        test_tokens("-NaN", &[Token::new(0..4, TokenKind::Float(-f64::NAN))]);

2072

1

        test_tokens("+inf", &[Token::new(0..4, TokenKind::Float(f64::INFINITY))]);

2073

1

        test_tokens("+NaN", &[Token::new(0..4, TokenKind::Float(f64::NAN))]);

2074

1

2075

2076

    #[test]

2077

1

    fn maps() {

2078

1

        test_tokens(

2079

1

            "{a:1,b:2}",

2080

1

&[

2081

1

                Token::new(0..1, TokenKind::Open(Balanced::Brace)),

2082

1

                Token::new(1..2, TokenKind::Identifier("a")),

2083

1

                Token::new(2..3, TokenKind::Colon),

2084

1

                Token::new(3..4, TokenKind::Integer(Integer::Usize(1))),

2085

1

                Token::new(4..5, TokenKind::Comma),

2086

1

                Token::new(5..6, TokenKind::Identifier("b")),

2087

1

                Token::new(6..7, TokenKind::Colon),

2088

1

                Token::new(7..8, TokenKind::Integer(Integer::Usize(2))),

2089

1

                Token::new(8..9, TokenKind::Close(Balanced::Brace)),

2090

1

],

2091

1

);

2092

1

2093

2094

    #[test]

2095

1

    fn strings() {

2096

        macro_rules! test_string {

2097

            ($char:tt) => {

2098

                let ch = core::stringify!($char);

2099

                test_tokens(

2100

ch,

2101

                    &[Token::new(

2102

                        0..ch.len(),

2103

                        TokenKind::String(Cow::Borrowed($char)),

2104

)],

2105

);

2106

};

2107

2108

1

        test_string!("");

2109

1

        test_string!("abc");

2110

1

        test_string!("\r\t\n\0\x41\u{1_F980}\\\"");

2111

1

        // string-continue, better tested with an escaped literal than trust the

2112

1

        // line endings being preserved in git.

2113

1

        test_tokens(

2114

1

            "\"a\\\n \t \r \n  b\"",

2115

1

            &[Token::new(0..14, TokenKind::String(Cow::Borrowed("ab")))],

2116

1

);

2117

1

2118

2119

    #[test]

2120

1

    fn raw_strings() {

2121

        macro_rules! test_string {

2122

            ($char:tt) => {

2123

                let ch = core::stringify!($char);

2124

                test_tokens(

2125

ch,

2126

                    &[Token::new(

2127

                        0..ch.len(),

2128

                        TokenKind::String(Cow::Borrowed($char)),

2129

)],

2130

);

2131

};

2132

2133

1

        test_string!(r###""##"###);

2134

1

        test_string!(r"abc");

2135

1

        test_string!(r#"abc"#);

2136

1

        test_string!(r#""\r\t\n\0\x41\u{1_F980}\\\"""#);

2137

1

2138

2139

    #[test]

2140

1

    fn chars() {

2141

        macro_rules! test_char {

2142

            ($char:tt) => {

2143

                let ch = core::stringify!($char);

2144

                test_tokens(ch, &[Token::new(0..ch.len(), TokenKind::Character($char))]);

2145

};

2146

2147

2148

1

        test_char!('\0');

2149

1

        test_char!('\r');

2150

1

        test_char!('\t');

2151

1

        test_char!('\\');

2152

1

        test_char!('\'');

2153

1

        test_char!('\"');

2154

1

        test_char!('\x42');

2155

1

        test_char!('\u{1_F980}');

2156

1

2157

2158

    #[test]

2159

1

    fn bytes() {

2160

        macro_rules! test_byte {

2161

            ($char:tt) => {

2162

                let ch = core::stringify!($char);

2163

                test_tokens(ch, &[Token::new(0..ch.len(), TokenKind::Byte($char))]);

2164

};

2165

2166

2167

1

        test_byte!(b'\0');

2168

1

        test_byte!(b'\r');

2169

1

        test_byte!(b'\t');

2170

1

        test_byte!(b'\\');

2171

1

        test_byte!(b'\'');

2172

1

        test_byte!(b'\"');

2173

1

        test_byte!(b'\x42');

2174

1

2175

2176

    #[test]

2177

1

    fn byte_strings() {

2178

        macro_rules! test_byte_string {

2179

            ($char:tt) => {

2180

                let ch = core::stringify!($char);

2181

                test_tokens(

2182

ch,

2183

                    &[Token::new(

2184

                        0..ch.len(),

2185

                        TokenKind::Bytes(Cow::Borrowed($char)),

2186

)],

2187

);

2188

};

2189

2190

2191

1

        test_byte_string!(b"hello world");

2192

1

        test_byte_string!(b"\0");

2193

1

        test_byte_string!(b"\r");

2194

1

        test_byte_string!(b"\t");

2195

1

        test_byte_string!(b"\\");

2196

1

        test_byte_string!(b"\'");

2197

1

        test_byte_string!(b"\"");

2198

1

        test_byte_string!(b"\x42");

2199

1

        // string-continue, better tested with an escaped literal than trust the

2200

1

        // line endings being preserved in git.

2201

1

        test_tokens(

2202

1

            "b\"a\\\n \t \r \n  b\"",

2203

1

            &[Token::new(0..15, TokenKind::Bytes(Cow::Borrowed(b"ab")))],

2204

1

);

2205

1

2206

2207

    #[test]

2208

1

    fn raw_byte_strings() {

2209

        macro_rules! test_string {

2210

            ($char:tt) => {

2211

                let ch = core::stringify!($char);

2212

                test_tokens(

2213

ch,

2214

                    &[Token::new(

2215

                        0..ch.len(),

2216

                        TokenKind::Bytes(Cow::Borrowed($char)),

2217

)],

2218

);

2219

};

2220

2221

1

        test_string!(br"abc");

2222

1

        test_string!(br###""##"###);

2223

1

        test_string!(br#"abc"#);

2224

1

        test_string!(br#""\r\t\n\0\x41\u{1_F980}\\\"""#);

2225

1

2226

2227

    #[test]

2228

1

    fn block_comments() {

2229

        macro_rules! test_comment {

2230

            ($comment:tt) => {

2231

                test_tokens_full(

2232

                    $comment,

2233

                    &[Token::new(0..$comment.len(), TokenKind::Comment($comment))],

2234

);

2235

};

2236

2237

1

        test_comment!("/* hello */");

2238

1

        test_comment!("/*** /* hello */ **/");

2239

1

2240

2241

    #[test]

2242

1

    fn single_line_comments() {

2243

1

        test_tokens_full(

2244

1

            "// test",

2245

1

            &[Token::new(0..7, TokenKind::Comment("// test"))],

2246

1

);

2247

1

        test_tokens_full(

2248

1

            "// test\n",

2249

1

&[

2250

1

                Token::new(0..7, TokenKind::Comment("// test")),

2251

1

                Token::new(7..8, TokenKind::Whitespace("\n")),

2252

1

],

2253

1

);

2254

1

2255