Split a text into BUFFER_MAX sized packets by preferably splitting at `\n`, or space.

bufsplit.rs

//! Split a text into BUFFER_MAX sized packets, by preferably splitting at `\n`, or space.
//! If that's not possible then the lines are cut at the last character before the buffer
//! size, keeping multibyte UTF-8 characters valid.
//! 
//! May contain nuts, slop (mainly the tests) and dairy products 
//! but it has been manually (eyebally?) reviewed.

pub const BUFFER_MAX: usize = 32_768;

/// Split `input` into `BUFFER_MAX` sized buffers.
/// Split priority per packet:
///   1. Last `\n` in [N/2-1, N-1)  — consumed, not emitted
///   2. Last whitespace in [0, N-1) — consumed, not emitted
///   3. Last UTF-8 char boundary at or before N-1
pub fn text_breaker(input: &str) -> Vec<&str> {
    text_breaker_n::<BUFFER_MAX>(input)
}

/// Splits `input` into packets of at most `N-1` content bytes each.
/// Split priority per packet:
///   1. Last `\n` in [N/2-1, N-1)  — consumed, not emitted
///   2. Last whitespace in [0, N-1) — consumed, not emitted
///   3. Last UTF-8 char boundary at or before N-1
///
/// Panics if N < 5.
pub fn text_breaker_n<const N: usize>(input: &str) -> Vec<&str> {
    assert!(N >= 5, "N must be at least 5 to accommodate any single UTF-8 codepoint (up to 4 bytes)");

    let content_max = N - 1; // bytes available for content
    let half = N / 2 - 1;   // start of preferred \n search zone

    let mut packets = Vec::new();
    let mut remaining = input;

    while !remaining.is_empty() {
        if remaining.len() <= content_max {
            packets.push(remaining);
            break;
        }

        // Position of the last unicode character
        let p = remaining.floor_char_boundary(content_max); // MSRV 1.91.0
        // Last unicode char before `half`
        let zone_start = remaining.floor_char_boundary(half);

        let (pos, consume) = if p > zone_start {
            // 1. split at `\n` (half,p)
            let zone = &remaining[zone_start..p];
            if let Some(nl_pos) = zone.rfind('\n') {
                Some((zone_start + nl_pos, true))
            } else {
                // 2. Split at whitespace (half,p)
                zone.rfind(|c: char| c.is_whitespace() && c != '\n')
                    .map(|ws_pos| (zone_start + ws_pos, true))
            }
        } else {
            None
        }
        .or_else(|| {
            // 2b. Split at whitespace (start, half)
            remaining[..zone_start]
                .rfind(|c: char| c.is_whitespace() && c != '\n')
                .map(|ws_pos| (ws_pos, true))
        })
        // 3. Hard cut at last UTF-8 boundary (which is p)
        .unwrap_or((p, false)); 

        packets.push(&remaining[..pos]);
        remaining = &remaining[pos + consume as usize..];
    }

    packets
}

// ------------------------------ tests ------------------------------------

#[cfg(test)]
mod tests {
    use super::*;

    // Convenience: convert packets to vec of strings for readable assertions
    fn to_strings(packets: &[&str]) -> Vec<String> {
        packets.iter().map(|&s| s.to_string()).collect()
    }

    // Verify no packet exceeds N-1 content bytes (caller adds any prefix)
    fn assert_invariants<const N: usize>(packets: &[&str]) {
        for (i, p) in packets.iter().enumerate() {
            assert!(p.len() < N, "packet {i} exceeds content_max={}: len={}", N - 1, p.len());
        }
    }

    // Reconstruct the original text from packets.
    // \n and spaces used as split points are consumed so we cannot recover them;
    // we only verify content bytes are preserved in order.
    fn content_bytes(packets: &[&str]) -> Vec<u8> {
        packets.iter().flat_map(|s| s.as_bytes().iter().copied()).collect()
    }

    // -------------------------------------------------------------------------
    // 1. Empty input
    // -------------------------------------------------------------------------
    #[test]
    fn test_empty() {
        let result = text_breaker_n::<8>("");
        assert!(result.is_empty());
    }

    // -------------------------------------------------------------------------
    // 2. Input fits exactly in one packet (content == content_max)
    // N=8 → content_max=7; input is exactly 7 bytes, no split chars
    // -------------------------------------------------------------------------
    #[test]
    fn test_fits_exactly_one_packet() {
        // N=8, content_max=7
        let input = "abcdefg"; // 7 bytes
        let packets = text_breaker_n::<8>(input);
        let s = to_strings(&packets);
        assert_invariants::<8>(&packets);
        assert_eq!(s, vec!["abcdefg"]);
    }

    // -------------------------------------------------------------------------
    // 3. Input shorter than half — single short packet
    // N=16 → half=7; input is 4 bytes
    // -------------------------------------------------------------------------
    #[test]
    fn test_shorter_than_half() {
        let input = "hi!";
        let packets = text_breaker_n::<16>(input);
        assert_invariants::<16>(&packets);
        assert_eq!(to_strings(&packets), vec!["hi!"]);
    }

    // -------------------------------------------------------------------------
    // 4. Split on \n in preferred zone [half, content_max)
    // N=16 → content_max=15, half=7
    // "aaaaaaaa\nbbbb" — \n at pos 8 which is >= half(7): split there
    // -------------------------------------------------------------------------
    #[test]
    fn test_split_on_newline_in_preferred_zone() {
        // N=16, half=7, content_max=15
        // "aaaaaaaa\nbbbbbbbbbbbbbbb" total > content_max
        // workbuf = "aaaaaaaa\nbbbbb" (15 bytes)
        // last \n in [7..15) is at pos 8 → split there
        let input = "aaaaaaaa\nbbbbbbbbbbbbbbb";
        let packets = text_breaker_n::<16>(input);
        assert_invariants::<16>(&packets);
        let s = to_strings(&packets);
        assert_eq!(s[0], "aaaaaaaa");
        assert_eq!(s[1], "bbbbbbbbbbbbbbb");
    }

    // -------------------------------------------------------------------------
    // 5. \n exists but only before half — falls through to space
    // N=16 → half=7, content_max=15
    // \n at pos 3 (< half=7), space at pos 10 → split at space
    // -------------------------------------------------------------------------
    #[test]
    fn test_newline_before_half_falls_to_space() {
        // workbuf (15 bytes): "aaa\naaaaaa bbb"
        //                     0123456789012345
        // \n at 3 < half(7): not in zone
        // space at 10: split there
        let input = "aaa\naaaaaa bbbbbbbbbbbbb";
        let packets = text_breaker_n::<16>(input);
        assert_invariants::<16>(&packets);
        let s = to_strings(&packets);
        assert_eq!(s[0], "aaa\naaaaaa");
        assert_eq!(s[1], "bbbbbbbbbbbbb");
    }

    // -------------------------------------------------------------------------
    // 6. Split on last space, no \n in preferred zone
    // N=16 → content_max=15, half=7
    // No \n anywhere, space at pos 9 → split at space
    // -------------------------------------------------------------------------
    #[test]
    fn test_split_on_last_space() {
        // workbuf: "aaaaaaaaa bbbbbb" would be 16, so workbuf="aaaaaaaaa bbbbb"(15)
        // space at pos 9 → split
        let input = "aaaaaaaaa bbbbbbbbbbbbbb";
        let packets = text_breaker_n::<16>(input);
        assert_invariants::<16>(&packets);
        let s = to_strings(&packets);
        assert_eq!(s[0], "aaaaaaaaa");
        assert_eq!(s[1], "bbbbbbbbbbbbbb");
    }

    // -------------------------------------------------------------------------
    // 7. Hard cut at char boundary — no \n, no space, pure ASCII
    // N=8 → content_max=7
    // "abcdefghij" → first packet "abcdefg", second "hij"
    // -------------------------------------------------------------------------
    #[test]
    fn test_hard_cut_ascii() {
        let input = "abcdefghij";
        let packets = text_breaker_n::<8>(input);
        assert_invariants::<8>(&packets);
        let s = to_strings(&packets);
        assert_eq!(s[0], "abcdefg");
        assert_eq!(s[1], "hij");
    }

    // 7b, there is \n but too early
    #[test]
    fn test_hard_cut_ascii_n_early() {
        let input = "ab\ncdefghij";
        let packets = text_breaker_n::<8>(input);
        assert_invariants::<8>(&packets);
        let s = to_strings(&packets);
        assert_eq!(s[0], "ab\ncdef");
        assert_eq!(s[1], "ghij");
    }

    // -------------------------------------------------------------------------
    // 8. Unicode: emoji+modifier straddling the hard cut boundary
    // "👨‍👩" is a ZWJ sequence: 👨=4 bytes, ZWJ=3 bytes, 👩=4 bytes = 11 bytes total
    // N=8 → content_max=7; hard cut must not split inside the codepoint
    // workbuf[..7] would cut inside 👨 (4 bytes) — back up to 0... 
    // actually let's use: "aa" + "👨‍👩" = 2 + 11 = 13 bytes
    // N=8: workbuf=7 bytes = "aa" + 👨(4b) + ZWJ[0] — cut must back up to pos 2
    // -------------------------------------------------------------------------
    #[test]
    fn test_unicode_hard_cut_no_split_inside_codepoint() {
        // "aa👨‍👩" = 2 + 4 + 3 + 4 = 13 bytes
        // N=8: content_max=7, no \n, no space
        // workbuf bytes 0..7: 'a','a', 4 bytes of 👨, 1st byte of ZWJ
        // last_char_boundary backs up: ZWJ continuation? No — ZWJ is E2 80 8D
        // E2 is a leading byte (3-byte seq), so pos=6 is continuation(80),
        // pos=5 is continuation(8D)... wait let me be precise:
        // 👨 = F0 9F 91 A8 (4 bytes)
        // ZWJ = E2 80 8D   (3 bytes)
        // 👩 = F0 9F 91 A9 (4 bytes)
        // workbuf[0..7] = 61 61 F0 9F 91 A8 E2
        //                0  1  2  3  4  5  6
        // bytes[6]=E2 is a leading byte (not continuation) so boundary is at 6
        // → packet 0 content = "aa👨" (6 bytes) ✓
        let input = "aa\u{1F468}\u{200D}\u{1F469}rest";
        let packets = text_breaker_n::<8>(input);
        assert_invariants::<8>(&packets);
        // All content bytes preserved (minus no consumed split chars)
        let all: Vec<u8> = content_bytes(&packets);
        assert_eq!(all, input.as_bytes());
    }

    // -------------------------------------------------------------------------
    // 8b. Emoji straddling exactly at content_max with continuation bytes
    // "aaa" + 🏳️‍🌈 (rainbow flag: F0 9F 8F B3 EF B8 8F E2 80 8D F0 9F 8C 88 = 13b)
    // N=8: content_max=7, workbuf[0..7]= 'a'*3 + F0 9F 8F B3 — cut inside emoji
    // last_char_boundary must back up to 3
    // -------------------------------------------------------------------------
    #[test]
    fn test_unicode_hard_cut_backs_up_to_before_multibyte() {
        // 🏳️‍🌈 = F0 9F 8F B3 EF B8 8F E2 80 8D F0 9F 8C 88 (13 bytes)
        let input = "aaa\u{1F3F3}\u{FE0F}\u{200D}\u{1F308}end";
        let packets = text_breaker_n::<8>(input);
        assert_invariants::<8>(&packets);
        assert_eq!(content_bytes(&packets), input.as_bytes());
    }

    // -------------------------------------------------------------------------
    // 9. Multiple packets chained — verify full content reconstruction
    // -------------------------------------------------------------------------
    #[test]
    fn test_multiple_packets_content_preserved() {
        // N=8, content_max=7; input has \n splits and hard cuts
        let input = "hello\nworld\nfoo";
        let packets = text_breaker_n::<8>(input);
        assert_invariants::<8>(&packets);
        // \n consumed, so reconstructed content won't have \n
        let all = content_bytes(&packets);
        let expected: Vec<u8> = input.bytes().filter(|&b| b != b'\n').collect();
        assert_eq!(all, expected);
    }

    // -------------------------------------------------------------------------
    // 10. Consecutive \n don't split
    // -------------------------------------------------------------------------
    #[test]
    fn test_consecutive_newlines() {
        let input = "a\n\nb";
        let packets = text_breaker_n::<8>(input);
        assert_invariants::<8>(&packets);
        let s = to_strings(&packets);
        assert_eq!(s[0], "a\n\nb");
    }

    // -------------------------------------------------------------------------
    // 11. \n as very last byte of workbuf (pos = content_max - 1)
    // N=8, content_max=7, half=3
    // input longer than 7 bytes, \n at exactly pos 6 (content_max-1)
    // -------------------------------------------------------------------------
    #[test]
    fn test_newline_at_last_position_of_workbuf() {
        // workbuf = "abcdef\n" (7 bytes), \n at pos 6 = content_max-1
        // half=3, zone=[3,7): \n at 6 is in zone → split at 6, consume \n
        let input = "abcdef\nXXXXXXX";
        let packets = text_breaker_n::<8>(input);
        assert_invariants::<8>(&packets);
        let s = to_strings(&packets);
        assert_eq!(s[0], "abcdef");
        assert_eq!(s[1], "XXXXXXX");
    }

    // -------------------------------------------------------------------------
    // 12. Space as very last byte of workbuf (pos = content_max - 1), no \n in zone
    // N=8, content_max=7
    // workbuf = "abcdef " (7 bytes), space at pos 6
    // -------------------------------------------------------------------------
    #[test]
    fn test_space_at_last_position_of_workbuf() {
        // no \n anywhere, space at pos 6 (last in workbuf)
        let input = "abcdef XXXXXXX";
        let packets = text_breaker_n::<8>(input);
        assert_invariants::<8>(&packets);
        let s = to_strings(&packets);
        assert_eq!(s[0], "abcdef");
        assert_eq!(s[1], "XXXXXXX");
    }

    // -------------------------------------------------------------------------
    // 13. Space only at position 0
    // N=8, content_max=7: workbuf = " abcdef", space only at 0
    // last space = 0 → content = "" (empty), next packet starts at "abcdef..."
    // -------------------------------------------------------------------------
    #[test]
    fn test_space_only_at_position_zero() {
        let input = " abcdefXXXXXXX";
        let packets = text_breaker_n::<8>(input);
        assert_invariants::<8>(&packets);
        let s = to_strings(&packets);
        assert_eq!(s[0], ""); // empty content, space consumed
        assert_eq!(s[1], "abcdefX");
        assert_eq!(s[2], "XXXXXX");
    }

    // -------------------------------------------------------------------------
    // 14. \n at exactly HALF (boundary inclusivity)
    // N=16, half=7, content_max=15
    // \n at pos 7 = half: should be found by zone search [half..)
    // -------------------------------------------------------------------------
    #[test]
    fn test_newline_at_exactly_half() {
        // workbuf (15 bytes): "aaaaaaa\nbbbbbbb" — \n at pos 7 = half
        // 15 b's so the tail fits exactly in one packet (content_max = 15)
        let input = "aaaaaaa\nbbbbbbbbbbbbbbb";
        let packets = text_breaker_n::<16>(input);
        assert_invariants::<16>(&packets);
        let s = to_strings(&packets);
        assert_eq!(s[0], "aaaaaaa");
        assert_eq!(s[1], "bbbbbbbbbbbbbbb");
    }

    // -------------------------------------------------------------------------
    // 15. \n at exactly HALF - 1 (just outside preferred zone)
    // N=16, half=7: \n at pos 6 is NOT in zone → falls to space or hard cut
    // -------------------------------------------------------------------------
    #[test]
    fn test_newline_just_before_half_falls_through() {
        // "aaaaaa\n" at pos 6, then no space, no \n in [7,15)
        // → hard cut at content_max=15 (all ASCII so boundary=15)
        let input = "aaaaaa\nbbbbbbbbbbbbbbbb";
        let packets = text_breaker_n::<16>(input);
        assert_invariants::<16>(&packets);
        let s = to_strings(&packets);
        // \n at 6 is below half(7): not used as preferred split
        // no space in workbuf → hard cut at content_max=15
        assert_eq!(s[0], "aaaaaa\nbbbbbbbb"); // 15 bytes of content
    }
}