Created
December 17, 2022 21:47
-
-
Save Cactiw/c2397561966e2f343b0563a6c5b7f17f to your computer and use it in GitHub Desktop.
This is quick fix for sentry generated backup.json file.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import json | |
| FILENAME = "backup.json" | |
| FILENAME_FIXED = "backup_fixed.json" | |
| def restore(): | |
| with open(FILENAME, "r") as f: | |
| lines = f.readlines() | |
| lines = lines[lines.index('[\n'):] | |
| with open(FILENAME_FIXED, "w") as f: | |
| f.writelines(lines) | |
| with open(FILENAME_FIXED, "r") as f: | |
| data = json.load(f) | |
| for d in data: | |
| value = d.get("fields", {}).get("value") | |
| if value is not None and not isinstance(value, str): | |
| d["fields"]["value"] = str(value) | |
| with open(FILENAME_FIXED, "w") as f: | |
| f.write(json.dumps(data, indent=4, ensure_ascii=False)) | |
| return data | |
| if __name__ == "__main__": | |
| restore() |
Still have some problems:
6:15:43 [INFO] sentry.plugins.github: apps-not-configured
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 91, in inner
return func(self, sql, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/base.py", line 86, in execute
return self.cursor.execute(sql, clean_bad_params(params))
psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "sentry_option_key_key"
DETAIL: Key (key)=(sentry:last_worker_ping) already exists.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 77, in inner
raise_the_exception(self.db, e)
File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 75, in inner
return func(self, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 18, in inner
return func(self, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 95, in inner
raise exc_info[0](msg).with_traceback(exc_info[2])
File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 91, in inner
return func(self, sql, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/base.py", line 86, in execute
return self.cursor.execute(sql, clean_bad_params(params))
psycopg2.errors.UniqueViolation: UniqueViolation('duplicate key value violates unique constraint "sentry_option_key_key"\nDETAIL: Key (key)=(sentry:last_worker_ping) already exists.\n')
SQL: UPDATE "sentry_option" SET "key" = %s, "value" = %s, "last_updated" = %s WHERE "sentry_option"."id" = %s
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/bin/sentry", line 8, in <module>
sys.exit(main())
File "/usr/local/lib/python3.8/site-packages/sentry/runner/__init__.py", line 192, in main
func(**kwargs)
File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1128, in __call__
return self.main(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1053, in main
rv = self.invoke(ctx)
File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1659, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1395, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/usr/local/lib/python3.8/site-packages/click/core.py", line 754, in invoke
return __callback(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/click/decorators.py", line 26, in new_func
return f(get_current_context(), *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/sentry/runner/decorators.py", line 29, in inner
return ctx.invoke(f, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/click/core.py", line 754, in invoke
return __callback(*args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/sentry/runner/commands/backup.py", line 21, in import_
obj.save()
File "/usr/local/lib/python3.8/site-packages/django/core/serializers/base.py", line 223, in save
models.Model.save_base(self.object, using=using, raw=True, **kwargs)
File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 780, in save_base
updated = self._save_table(
File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 853, in _save_table
updated = self._do_update(base_qs, using, pk_val, values, update_fields,
File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 903, in _do_update
return filtered._update(values) > 0
File "/usr/local/lib/python3.8/site-packages/django/db/models/query.py", line 760, in _update
return query.get_compiler(self.db).execute_sql(CURSOR)
File "/usr/local/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1471, in execute_sql
cursor = super().execute_sql(result_type)
File "/usr/local/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1142, in execute_sql
cursor.execute(sql, params)
File "/usr/local/lib/python3.8/site-packages/sentry_sdk/integrations/django/__init__.py", line 563, in execute
return real_execute(self, sql, params)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 67, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers
return executor(sql, params, many, context)
File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/lib/python3.8/site-packages/django/db/utils.py", line 89, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql, params)
File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 77, in inner
raise_the_exception(self.db, e)
File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 75, in inner
return func(self, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 18, in inner
return func(self, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 95, in inner
raise exc_info[0](msg).with_traceback(exc_info[2])
File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 91, in inner
return func(self, sql, *args, **kwargs)
File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/base.py", line 86, in execute
return self.cursor.execute(sql, clean_bad_params(params))
django.db.utils.IntegrityError: UniqueViolation('duplicate key value violates unique constraint "sentry_option_key_key"\nDETAIL: Key (key)=(sentry:last_worker_ping) already exists.\n')
SQL: UPDATE "sentry_option" SET "key" = %s, "value" = %s, "last_updated" = %s WHERE "sentry_option"."id" = %s
Still have some problems:
6:15:43 [INFO] sentry.plugins.github: apps-not-configured Traceback (most recent call last): File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 91, in inner return func(self, sql, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/base.py", line 86, in execute return self.cursor.execute(sql, clean_bad_params(params)) psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "sentry_option_key_key" DETAIL: Key (key)=(sentry:last_worker_ping) already exists. During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute return self.cursor.execute(sql, params) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 77, in inner raise_the_exception(self.db, e) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 75, in inner return func(self, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 18, in inner return func(self, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 95, in inner raise exc_info[0](msg).with_traceback(exc_info[2]) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 91, in inner return func(self, sql, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/base.py", line 86, in execute return self.cursor.execute(sql, clean_bad_params(params)) psycopg2.errors.UniqueViolation: UniqueViolation('duplicate key value violates unique constraint "sentry_option_key_key"\nDETAIL: Key (key)=(sentry:last_worker_ping) already exists.\n') SQL: UPDATE "sentry_option" SET "key" = %s, "value" = %s, "last_updated" = %s WHERE "sentry_option"."id" = %s The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/usr/local/bin/sentry", line 8, in <module> sys.exit(main()) File "/usr/local/lib/python3.8/site-packages/sentry/runner/__init__.py", line 192, in main func(**kwargs) File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1128, in __call__ return self.main(*args, **kwargs) File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1053, in main rv = self.invoke(ctx) File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1659, in invoke return _process_result(sub_ctx.command.invoke(sub_ctx)) File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1395, in invoke return ctx.invoke(self.callback, **ctx.params) File "/usr/local/lib/python3.8/site-packages/click/core.py", line 754, in invoke return __callback(*args, **kwargs) File "/usr/local/lib/python3.8/site-packages/click/decorators.py", line 26, in new_func return f(get_current_context(), *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/runner/decorators.py", line 29, in inner return ctx.invoke(f, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/click/core.py", line 754, in invoke return __callback(*args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/runner/commands/backup.py", line 21, in import_ obj.save() File "/usr/local/lib/python3.8/site-packages/django/core/serializers/base.py", line 223, in save models.Model.save_base(self.object, using=using, raw=True, **kwargs) File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 780, in save_base updated = self._save_table( File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 853, in _save_table updated = self._do_update(base_qs, using, pk_val, values, update_fields, File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 903, in _do_update return filtered._update(values) > 0 File "/usr/local/lib/python3.8/site-packages/django/db/models/query.py", line 760, in _update return query.get_compiler(self.db).execute_sql(CURSOR) File "/usr/local/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1471, in execute_sql cursor = super().execute_sql(result_type) File "/usr/local/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1142, in execute_sql cursor.execute(sql, params) File "/usr/local/lib/python3.8/site-packages/sentry_sdk/integrations/django/__init__.py", line 563, in execute return real_execute(self, sql, params) File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 67, in execute return self._execute_with_wrappers(sql, params, many=False, executor=self._execute) File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers return executor(sql, params, many, context) File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute return self.cursor.execute(sql, params) File "/usr/local/lib/python3.8/site-packages/django/db/utils.py", line 89, in __exit__ raise dj_exc_value.with_traceback(traceback) from exc_value File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute return self.cursor.execute(sql, params) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 77, in inner raise_the_exception(self.db, e) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 75, in inner return func(self, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 18, in inner return func(self, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 95, in inner raise exc_info[0](msg).with_traceback(exc_info[2]) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 91, in inner return func(self, sql, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/base.py", line 86, in execute return self.cursor.execute(sql, clean_bad_params(params)) django.db.utils.IntegrityError: UniqueViolation('duplicate key value violates unique constraint "sentry_option_key_key"\nDETAIL: Key (key)=(sentry:last_worker_ping) already exists.\n') SQL: UPDATE "sentry_option" SET "key" = %s, "value" = %s, "last_updated" = %s WHERE "sentry_option"."id" = %s
It seems like your target database is not empty, maybe Sentry has already recorded something there?
I also had to remove every sentry.useremail block from the backup file.
Error while importing to an empty database:
django.db.utils.IntegrityError: UniqueViolation('duplicate key value violates unique constraint "sentry_useremail_user_id_email_ade975f1_uniq"\nDETAIL: Key (user_id, email)=(3, [email protected]) already exists.\n')
SQL: UPDATE "sentry_useremail" SET "user_id" = %s, "email" = %s, "validation_hash" = %s, "date_hash_added" = %s, "is_verified" = %s WHERE "sentry_useremail"."id" = %s
Email addresses are probably already added by the sentry.user backups...
This script caused us issues with boolean values being converted to uppercase string "False". The import didn't like them.
I've adapted the script to account for this:
import json
FILENAME = "backup.json"
FILENAME_FIXED = "backup_fixed_2.json"
def restore():
with open(FILENAME, "r") as f:
lines = f.readlines()
lines = lines[lines.index('[\n'):]
with open(FILENAME_FIXED, "w") as f:
f.writelines(lines)
with open(FILENAME_FIXED, "r") as f:
data = json.load(f)
for d in data:
value = d.get("fields", {}).get("value")
if isinstance(value, bool):
d["fields"]["value"] = str(value).lower()
elif value is not None and not isinstance(value, str):
d["fields"]["value"] = str(value)
with open(FILENAME_FIXED, "w") as f:
f.write(json.dumps(data, indent=4, ensure_ascii=False))
return data
if __name__ == "__main__":
restore()
This script is broken and imports booleans as (encoded) strings into the database.
A True should be encoded as "gAKILg==" but gets inserted as "gAJYBAAAAFRydWVxAC4=" into the database.
After hours of debugging i finally managed to fix my installation with multiple manual db queries.
Sadly i dont have every query in my history, but some are here:
update sentry_option set value = 'gAKILg==' where value = 'gAJYBAAAAFRydWVxAC4=';
update sentry_organizationoptions set value = 'gAKILg==' where value = 'gAJYBAAAAFRydWVxAC4=';
update sentry_projectoptions set value = 'gAKILg==' where value = 'gAJYBAAAAFRydWVxAC4=';
@Lucaber , my adapted script above should be of help if you have your original exported backup.
If i remember correctly, numbers were also saved incorrectly as base64 encoded strings with quotes. 8 -> "8" | base64.
This resulted in broken "sentry:option-epoch" options.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Usage:
python3 sentry_backup_fix.pyCorrect file will be created with backup_fixed.json name