Last active
January 1, 2024 21:58
-
-
Save mcarrowd/e436b6f882897cdbfeb020453dc328f4 to your computer and use it in GitHub Desktop.
golang win1251 <-> utf8 encoding issue demonstration
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"fmt" | |
"golang.org/x/text/encoding/charmap" | |
) | |
func main() { | |
for i := 0; i <= 255; i++ { | |
charnum := uint8(i) | |
orig := []uint8{charnum} | |
utf8 := DecodeWindows1251(orig) | |
win1251 := EncodeWindows1251(utf8) | |
if string(orig) != string(win1251) { | |
fmt.Printf("Oh~.. Failed at charnum 0x%x\n", i) | |
fmt.Printf("(orig, utf8, win1251)\n") | |
fmt.Printf("string: %s, %s, %s\n", string(orig), string(utf8), string(win1251)) | |
fmt.Printf("quoted: %+q, %+q, %+q\n", string(orig), string(utf8), string(win1251)) | |
fmt.Printf("hex : % x, % x, % x\n", string(orig), string(utf8), string(win1251)) | |
} | |
} | |
} | |
func DecodeWindows1251(ba []uint8) []uint8 { | |
dec := charmap.Windows1251.NewDecoder() | |
out, _ := dec.Bytes(ba) | |
return out | |
} | |
func EncodeWindows1251(ba []uint8) []uint8 { | |
enc := charmap.Windows1251.NewEncoder() | |
out, _ := enc.String(string(ba)) | |
return []uint8(out) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment