Skip to content

Instantly share code, notes, and snippets.

@alingogo

alingogo/change_data.rb

Last active December 10, 2015 22:08
Show Gist options
  • Save alingogo/4499743 to your computer and use it in GitHub Desktop.
Save alingogo/4499743 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
change_data.rb
require 'ostruct'
require 'csv'
require 'rubygems'
require 'kdtree'
class GeoName < OpenStruct
end
class GeoNames
class << self
def parse(file)
File.new(file).each_line do |line|
g = GeoName.new
s = line.chomp.split("\t")
g.geonameid = s[0]
g.name = s[1]
g.asciiname = s[2]
g.alternatenames = s[3]
g.latitude = s[4]
g.longitude = s[5]
g.feature_class = s[6]
g.feature_code = s[7]
g.country_code = s[8]
g.cc2 = s[9]
g.admin1 = s[10]
g.admin2 = s[11]
g.admin3 = s[12]
g.admin4 = s[13]
g.population = s[14]
g.elevation = s[15]
g.gtopo30 = s[16]
g.timezone = s[17]
# g.modification_date = Time.parse(s[18])
yield g
end
end
end
end
class Timezones
class << self
def parse(file)
File.new(file).each_line do |line|
t = GeoName.new
s = line.chomp.split("\t")
unless s[0].to_s == "CountryCode"
t.countrycode = s[0]
t.timezoneid = s[1]
t.gmt_offset = s[2]
t.dst_offset = s[3]
t.raw_offset = s[4]
end
yield t
end
end
end
end
citiesfile = "data/allCountries.txt"
kdtreefile = "data/allCountries.kdtree"
idtzfile = "data/id_tz_all.csv"
####################
# geonameidを取得するために、Kdtree.new(f)用なファイルを生成する
###################
points = []
GeoNames.parse(citiesfile) do |place|
points << [place.latitude.to_f, place.longitude.to_f, place.geonameid.to_i]
end
kd = Kdtree.new(points)
File.open(kdtreefile, "w") {|f| kd.persist(f)}
###################################
# geonameidより、タイムゾーンを取得するために、geonameid(int), timezoneid
###################################
CSV.open(idtzfile, "wb") do |csv|
GeoNames.parse(citiesfile) do |place|
csv << [place.geonameid, place.timezone]
end
end
Raw
compaire_from_countries15000.rb
require 'ostruct'
require 'csv'
require 'date'
require 'net/http'
require 'uri'
require 'json'
require 'rubygems'
require 'kdtree'
require 'sqlite3'
class GeoName < OpenStruct
end
class GeoNames
class << self
def parse(file)
File.new(file).each_line do |line|
g = GeoName.new
s = line.chomp.split("\t")
g.geonameid = s[0]
g.name = s[1]
g.asciiname = s[2]
g.alternatenames = s[3]
g.latitude = s[4]
g.longitude = s[5]
g.feature_class = s[6]
g.feature_code = s[7]
g.country_code = s[8]
g.cc2 = s[9]
g.admin1 = s[10]
g.admin2 = s[11]
g.admin3 = s[12]
g.admin4 = s[13]
g.population = s[14]
g.elevation = s[15]
g.gtopo30 = s[16]
g.timezone = s[17]
# g.modification_date = Time.parse(s[18])
yield g
end
end
end
end
class Timezones
class << self
def parse(file)
File.new(file).each_line do |line|
t = GeoName.new
s = line.chomp.split("\t")
unless s[0].to_s == "CountryCode"
t.countrycode = s[0]
t.timezoneid = s[1]
t.gmt_offset = s[2]
t.dst_offset = s[3]
t.raw_offset = s[4]
end
yield t
end
end
end
end
class Tzin
def initialize(kdf, idtzf)
@tz = {}
CSV.foreach(idtzf) do |csv|
@tz[csv[0].to_i] = csv[1]
end
@kd = File.open(kdf) { |f| Kdtree.new(f) }
end
def timezoneid(lat, lng)
id = @kd.nearest(lat.to_f, lng.to_f)
@tz[id]
end
end
class Tzout
GEOURL = 'api.geonames.org'
USERNAME = 'alinspoon'
def self.timezoneid(lat, lng)
begin
if proxy = ENV["http_proxy"] || ENV["HTTP_PROXY"]
uri = URI.parse(proxy)
proxy_user, proxy_pass = uri.userinfo.split(/:/) if uri.userinfo
response = Net::HTTP.Proxy(uri.host, uri.port,
proxy_user, proxy_pass).start(GEOURL) do |http|
http.get("/timezoneJSON?lat=#{lat}&lng=#{lng}&username=#{USERNAME}").body
end
else
response = Net::HTTP.get(GEOURL, "/timezoneJSON?lat=#{lat}&lng=#{lng}&username=#{USERNAME}")
end
js = JSON.parse(response)
[js['timezoneId'], js['gmtOffset']]
rescue Exception => e
p e
end
end
end
def get_points
citiesfile = "data/cities15000.txt"
points = []
GeoNames.parse(citiesfile) do |place|
if rand(100) == 1
points << [place.latitude.to_i, place.longitude.to_i]
end
end
points
end
#def test_get_points
# points = get_points
#end
def test
allkd = "data/allCountries.kdtree"
allidtz = "data/id_tz_all.csv"
citykd = "data/cities15000.kdtree"
cityidtz = "data/id_tz.csv"
tzin_c = Tzin.new(citykd, cityidtz)
tzin_a = Tzin.new(allkd, allidtz)
errors_c = 0
errors_a = 0
errors_tz2nil = 0
points = get_points
p "points.size:"
p points.size
points.each do |p|
lat, lng = p
tz1_c = tzin_c.timezoneid(lat, lng)
tz1_a = tzin_a.timezoneid(lat, lng)
tz2, gmt = Tzout.timezoneid(lat, lng)
errors_c += 1 unless tz1_c == tz2
if tz2.nil?
ENV['TZ'] = tz1_a
errors_a += 1 unless gmt.to_i == Time.now.gmtoff/3600
elsif tz1_a != tz2
errors_a += 1
end
sleep 1
end
p "errors_c:"
p errors_c
p "errors_a:"
p errors_a
p "errors_tz2nil:"
p errors_tz2nil
end
test
Raw
comparison.rb
require 'csv'
require 'date'
require 'net/http'
require 'uri'
require 'json'
require 'rubygems'
require 'kdtree'
class Tzin
def initialize(kdf, idtzf)
@tz = {}
CSV.foreach(idtzf) do |csv|
@tz[csv[0].to_i] = csv[1]
end
@kd = File.open(kdf) { |f| Kdtree.new(f) }
end
def timezoneid(lat, lng)
id = @kd.nearest(lat.to_f, lng.to_f)
@tz[id]
end
end
class Tzout
GEOURL = 'api.geonames.org'
USERNAME = 'alinspoon'
def self.timezoneid(lat, lng)
begin
if proxy = ENV["http_proxy"] || ENV["HTTP_PROXY"]
uri = URI.parse(proxy)
proxy_user, proxy_pass = uri.userinfo.split(/:/) if uri.userinfo
response = Net::HTTP.Proxy(uri.host, uri.port,
proxy_user, proxy_pass).start(GEOURL) do |http|
http.get("/timezoneJSON?lat=#{lat}&lng=#{lng}&username=#{USERNAME}").body
end
else
response = Net::HTTP.get(GEOURL, "/timezoneJSON?lat=#{lat}&lng=#{lng}&username=#{USERNAME}")
end
js = JSON.parse(response)
[js['timezoneId'], js['gmtOffset']]
rescue Exception => e
p e
end
end
end
class GenRand
class << self
def latlng(ca)
case ca
when "ne"
[rand90, rand180]
when "nw"
[rand90, -rand180]
when "se"
[-rand90, rand180]
when "sw"
[-rand90, -rand180]
end
end
private
def rand90
Random.new.rand 89.9
end
def rand180
Random.new.rand 179.9
end
end
end
def test
allkd = "data/allCountries.kdtree"
allidtz = "data/id_tz_all.csv"
citykd = "data/cities15000.kdtree"
cityidtz = "data/id_tz.csv"
tzin_c = Tzin.new(citykd, cityidtz)
tzin_a = Tzin.new(allkd, allidtz)
errors_c = 0
errors_a = 0
errors_tz2nil = 0
kaisu = 100
kaisu.times do
["ne","nw","se","sw"].each do |p|
lat, lng = GenRand.latlng(p)
tz1_c = tzin_c.timezoneid(lat, lng)
tz1_a = tzin_a.timezoneid(lat, lng)
tz2, gmt = Tzout.timezoneid(lat, lng)
errors_c += 1 unless tz1_c == tz2
unless tz1_a == tz2
if tz2.nil?
ENV['TZ'] = tz1_a
errors_tz2nil += 1 unless gmt.to_i == Time.now.gmtoff/3600
end
errors_a += 1
end
sleep 1
end
end
p "errors_c:"
p errors_c
p "errors_a:"
p errors_a
p "errors_tz2nil:"
p errors_tz2nil
p "kaisu"
p kaisu*4
end
test
Raw
comparison_isan.rb
require 'csv'
require 'date'
require 'net/http'
require 'uri'
require 'json'
require 'rubygems'
require 'kdtree'
class Tzin
def initialize(kdf, idtzf)
@tz = {}
CSV.foreach(idtzf) do |csv|
@tz[csv[0].to_i] = [csv[1],csv[2],csv[3]]
end
@kd = File.open(kdf) { |f| Kdtree.new(f) }
end
def timezoneid(lat, lng)
id = @kd.nearest(lat.to_f, lng.to_f)
@tz[id]
end
end
class Tzout
GEOURL = 'api.geonames.org'
USERNAME = 'alinspoon'
def self.timezoneid(lat, lng)
begin
if proxy = ENV["http_proxy"] || ENV["HTTP_PROXY"]
uri = URI.parse(proxy)
proxy_user, proxy_pass = uri.userinfo.split(/:/) if uri.userinfo
response = Net::HTTP.Proxy(uri.host, uri.port,
proxy_user, proxy_pass).start(GEOURL) do |http|
http.get("/timezoneJSON?lat=#{lat}&lng=#{lng}&username=#{USERNAME}").body
end
else
response = Net::HTTP.get(GEOURL, "/timezoneJSON?lat=#{lat}&lng=#{lng}&username=#{USERNAME}")
end
js = JSON.parse(response)
[js['timezoneId'], js['gmtOffset']]
rescue Exception => e
p e
end
end
end
def test
allkd = "data/allCountries.kdtree"
allidtz = "data/id_tz.csv"
isanfile = "data/isan.csv"
tzin_a = Tzin.new(allkd, allidtz)
errors = 0
CSV.foreach(isanfile) do |csv|
lat, lng = csv[1], csv[2]
tz1_a, tz1_gmt, tz1_dst = tzin_a.timezoneid(lat, lng)
tz2, gmt = Tzout.timezoneid(lat, lng)
if tz1_a != tz2
unless tz1_a.nil? || tz1_a.empty?
ENV['TZ'] = tz1_a
unless gmt.to_i == Time.now.gmtoff/3600
errors += 1
puts "tz1_gmt: " + (Time.now.gmtoff/3600).to_s
puts "gmt: " + gmt.to_s
puts csv[0].to_s + " " + csv[3].to_s
puts "-----------------"
end
end
elsif tz1_a.nil?
unless gmt.to_i == tz1_gmt.to_i
errors += 1
puts "tz1_gmt: " + tz1_gmt.to_s
puts "gmt: " + gmt.to_s
puts csv[0].to_s + " " + csv[3].to_s
puts "-----------------"
end
end
sleep 0.3
end
p "errors:"
p errors
end
test
Raw
id_tz_data_hosei.rb
equire 'ostruct'
require 'csv'
require 'date'
require 'net/http'
require 'uri'
require 'json'
require 'rubygems'
require 'kdtree'
class GeoName < OpenStruct
end
class GeoNames
class << self
def parse(file)
File.new(file).each_line do |line|
g = GeoName.new
s = line.chomp.split("\t")
g.geonameid = s[0]
g.name = s[1]
g.asciiname = s[2]
g.alternatenames = s[3]
g.latitude = s[4]
g.longitude = s[5]
g.feature_class = s[6]
g.feature_code = s[7]
g.country_code = s[8]
g.cc2 = s[9]
g.admin1 = s[10]
g.admin2 = s[11]
g.admin3 = s[12]
g.admin4 = s[13]
g.population = s[14]
g.elevation = s[15]
g.gtopo30 = s[16]
g.timezone = s[17]
# g.modification_date = Time.parse(s[18])
yield g
end
end
end
end
class Timezones
class << self
def parse(file)
File.new(file).each_line do |line|
t = GeoName.new
s = line.chomp.split("\t")
unless s[0].to_s == "CountryCode"
t.countrycode = s[0]
t.timezoneid = s[1]
t.gmt_offset = s[2]
t.dst_offset = s[3]
t.raw_offset = s[4]
end
yield t
end
end
end
end
class Tzin
KDFILE = "data/allCountries.kdtree"
IDTZFILE = "data/id_tz.csv"
def initialize
@tz = {}
CSV.foreach(IDTZFILE) do |csv|
@tz[csv[0].to_i] = csv[1]
end
@kd = File.open(KDFILE) { |f| Kdtree.new(f) }
end
def timezoneid(lat, lng)
id = @kd.nearest(lat.to_f, lng.to_f)
@tz[id]
end
end
class Tzout
GEOURL = 'api.geonames.org'
USERNAME = 'alinspoon'
def self.timezoneid(lat, lng)
begin
if proxy = ENV["http_proxy"] || ENV["HTTP_PROXY"]
uri = URI.parse(proxy)
proxy_user, proxy_pass = uri.userinfo.split(/:/) if uri.userinfo
response = Net::HTTP.Proxy(uri.host, uri.port,
proxy_user, proxy_pass).start(GEOURL) do |http|
http.get("/timezoneJSON?lat=#{lat}&lng=#{lng}&username=#{USERNAME}").body
end
else
response = Net::HTTP.get(GEOURL, "/timezoneJSON?lat=#{lat}&lng=#{lng}&username=#{USERNAME}")
end
js = JSON.parse(response)
[js['timezoneId'], js['gmtOffset'], js['dstOffset']]
rescue Exception => e
p e
end
end
end
citiesfile = "data/allCountries.txt"
kdtreefile = "data/allCountries.kdtree"
idtzfile = "data/id_tz.csv"
idtzhosei = "data/id_tz_hosei.csv"
####################
# geonameidを取得するために、Kdtree.new(f)用なファイルを生成する
###################
timesnil = 0
total = 0
tzoutnil = 0
p Time.now
CSV.open(idtzhosei, "wb") do |csv|
GeoNames.parse(citiesfile) do |place|
if place.timezone.nil? || place.timezone == ""
tz, gmtoff, dstoff = Tzout.timezoneid(place.latitude, place.longitude)
if tz
csv << [place.geonameid, tz, gmtoff, dstoff]
end
timesnil += 1
else
csv << [place.geonameid, place.timezone, nil, nil]
end
total += 1
end
end
p Time.now
p tzoutnil
p timesnil
p total
Raw
local2utc.rb
require 'csv'
require 'date'
require 'rubygems'
require 'kdtree'
tmpfile = "data/cities15000.kdtree"
idtzfile = "data/id_tz.csv"
tz = {}
CSV.foreach(idtzfile) do |csv|
tz[csv[0].to_i] = csv[1]
end
kd = File.open(tmpfile) { |f| Kdtree.new(f) }
loop do
p "input lat or input q for quit"
lat = gets.chomp
if lat == "q"
"input over........"
break
else
p "input lng"
lng = gets.chomp
p "input localtime ex. 2000-1-1 15:00:00"
lt = gets.chomp
end
p id = kd.nearest(lat.to_f, lng.to_f)
p tz[id]
ENV['TZ'] = tz[id]
d = DateTime.parse(lt)
p Time.local(d.year, d.month, d.day, d.hour, d.min, d.sec).utc
end
Raw
test_allCountries_data.rb
require 'ostruct'
require 'csv'
require 'rubygems'
require 'kdtree'
class GeoName < OpenStruct
end
class GeoNames
class << self
def parse(file)
File.new(file).each_line do |line|
g = GeoName.new
s = line.chomp.split("\t")
g.geonameid = s[0]
g.name = s[1]
g.asciiname = s[2]
g.alternatenames = s[3]
g.latitude = s[4]
g.longitude = s[5]
g.feature_class = s[6]
g.feature_code = s[7]
g.country_code = s[8]
g.cc2 = s[9]
g.admin1 = s[10]
g.admin2 = s[11]
g.admin3 = s[12]
g.admin4 = s[13]
g.population = s[14]
g.elevation = s[15]
g.gtopo30 = s[16]
g.timezone = s[17]
# g.modification_date = Time.parse(s[18])
yield g
end
end
end
end
class Timezones
class << self
def parse(file)
File.new(file).each_line do |line|
t = GeoName.new
s = line.chomp.split("\t")
unless s[0].to_s == "CountryCode"
t.countrycode = s[0]
t.timezoneid = s[1]
t.gmt_offset = s[2]
t.dst_offset = s[3]
t.raw_offset = s[4]
end
yield t
end
end
end
end
citiesfile = "data/allCountries.txt"
kdtreefile = "data/allCountries.kdtree"
idtzfile = "data/id_tz.csv"
####################
# geonameidを取得するために、Kdtree.new(f)用なファイルを生成する
###################
timesnil = 0
total = 0
GeoNames.parse(citiesfile) do |place|
if place.timezone.nil? || place.timezone == ""
p place.geonameid
timesnil += 1
end
total += 1
end
p timesnil
p total
p timesnil.to_f/total
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment