Skip to content

Instantly share code, notes, and snippets.

@pfmiles

pfmiles/HttpInvoker.java

Last active September 27, 2020 09:14
Show Gist options
  • Save pfmiles/9077f0d6d407a7a149900e0eb96c0711 to your computer and use it in GitHub Desktop.
Save pfmiles/9077f0d6d407a7a149900e0eb96c0711 to your computer and use it in GitHub Desktop.
Download ZIP
短频快http api调用工具，鼓励长连接
Raw
HttpInvoker.java
package test;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import com.alibaba.fastjson.JSON;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpMessage;
import org.apache.http.HttpVersion;
import org.apache.http.NameValuePair;
import org.apache.http.NoHttpResponseException;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.config.ConnectionConfig;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.DefaultHostnameVerifier;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.util.PublicSuffixMatcherLoader;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultClientConnectionReuseStrategy;
import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HTTP;
import org.apache.http.ssl.SSLContexts;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* http调用工具, 默认的实现参数鼓励长连接
*
* @author pf-miles Sep 6, 2016 2:12:06 PM
*/
public class HttpInvoker {
// 重试次数
private static final int RETRY_TIMES = 3;
private static final int DFT_SO_TIMEOUT = 3000; // 默认的调用超时时间
private static final int DFT_CON_TIMEOUT = 5000; // 默认的连接建立超时时间
private static final CloseableHttpClient httpClient;
private static Boolean HTTPS_HOSTNAME_VERIFY_IGNORE = true; // https
private static Logger logger = LoggerFactory.getLogger(HttpInvoker.class);
static {
HttpClientBuilder builder = HttpClients.custom();
builder.disableCookieManagement();
builder.disableRedirectHandling();
builder.disableAutomaticRetries();// 避免自动重试
builder.setConnectionReuseStrategy(DefaultClientConnectionReuseStrategy.INSTANCE);
builder.setDefaultRequestConfig(RequestConfig.custom().setAuthenticationEnabled(false)
.setCircularRedirectsAllowed(false)
// socket数据read/write默认超时时间,ms
.setSocketTimeout(DFT_SO_TIMEOUT)
// 与服务器端建立连接的最长等待时间,ms
.setConnectTimeout(DFT_CON_TIMEOUT)
// 从连接池中取出连接的最长等待时间,ms
.setConnectionRequestTimeout(1000)
// 不支持重定向
.setRedirectsEnabled(false).setCookieSpec(CookieSpecs.IGNORE_COOKIES)// 忽略cookie
.build());
builder.setKeepAliveStrategy(DefaultConnectionKeepAliveStrategy.INSTANCE);
// builder.setSchemePortResolver(schemePortResolver)
// builder.setServiceUnavailableRetryStrategy(new DefaultServiceUnavailableRetryStrategy(0,
// 1000));// 对服务端错误不重试
builder.setUserAgent("HttpApiInvoker/1.0");
final int conExpire = 15;// 长连接闲置过期时间
final PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(
RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.getSocketFactory())
.register("https", createSslSocksFactory()).build(),
null, null, null, conExpire, TimeUnit.SECONDS);
cm.setDefaultConnectionConfig(ConnectionConfig.DEFAULT);
cm.setDefaultMaxPerRoute(1024);// 单个route的最大连接数
cm.setDefaultSocketConfig(SocketConfig.custom()
// 设置长连接心跳检测
.setSoKeepAlive(true)
// 调用超时
.setSoTimeout(DFT_SO_TIMEOUT)
// api调用大多是短频快，因此禁用naggle粘包算法
.setTcpNoDelay(true).build());
cm.setMaxTotal(8192);// 全局最大总链接数量
cm.setValidateAfterInactivity(-1);// 禁用连接活性检测，在每次取出重用的连接时, 略微提升性能
builder.setConnectionManager(cm);
builder.setConnectionManagerShared(false);
httpClient = builder.build();
// 主动staleCheck, 免去运行时staleCheck, conExpire秒闲置即可close
Thread staleCheckThread = new Thread(new Runnable() {
public void run() {
while (true) {
try {
Thread.sleep(7717);// 质数，降低运行时间点重合狗血几率
cm.closeExpiredConnections();
cm.closeIdleConnections(conExpire, TimeUnit.SECONDS);
} catch (Exception e) {
// ignore...
}
}
}
}, "HttpInvoker-connection-stale-check-thread") {
};
staleCheckThread.setDaemon(true);
staleCheckThread.start();
Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
public void run() {
try {
httpClient.close();
} catch (IOException e) {
}
}
}, "HttpInvoker-shutdown-thread"));
}
// 定制ssl行为
private static ConnectionSocketFactory createSslSocksFactory() {
return new SSLConnectionSocketFactory(SSLContexts.createDefault(),
HTTPS_HOSTNAME_VERIFY_IGNORE ? NoopHostnameVerifier.INSTANCE
: new DefaultHostnameVerifier(PublicSuffixMatcherLoader.getDefault()));
}
/**
* 发起post请求, 对连接类错误会自动重试，最多三次
*
* @param url 请求的url
* @param params 请求的参数, 值可以是string或object[]
* @param customHeaders 自定义header, 值可以是string或string[]
* @param conTimeout socket连接超时时间, ms
* @param readTimeout 业务调用超时时间, ms
* @param requestEncoding 发起请求时使用的编码格式
* @return response
* @throws Exception
*/
public static CloseableHttpResponse post(String url, Map<String, ?> params,
Map<String, Object> customHeaders, int conTimeout,
int readTimeout,
String requestEncoding) throws Exception {
HttpPost post = new HttpPost(url);
post.setProtocolVersion(HttpVersion.HTTP_1_1);
post.setHeader(HTTP.CONN_DIRECTIVE, HTTP.CONN_KEEP_ALIVE);
// set header
setCustomHeaders(post, customHeaders);
// set body
UrlEncodedFormEntity entity;
try {
entity = new UrlEncodedFormEntity(buildParamList(params), requestEncoding);
} catch (UnsupportedEncodingException e) {
throw e;
}
post.setEntity(entity);
// set config
post.setConfig(RequestConfig.custom().setConnectTimeout(conTimeout)
.setSocketTimeout(readTimeout).build());
// invoke
Exception retryableEx = null;
for (int i = 0; i < RETRY_TIMES; i++) {
try {
return httpClient.execute(post);
} catch (ClientProtocolException e) {
throw new Exception(
"Error posting url: " + url + ", params: " + JSON.toJSONString(params)
+ ", custom headers: " + JSON.toJSONString(customHeaders)
+ ", conTimeout: " + conTimeout + ", readTimeout: "
+ readTimeout + ", encoding: " + requestEncoding + ".",
e);
} catch (IOException e) {
if (retryable(e)) {
retryableEx = e;
} else {
throw new Exception(
"Error posting url: " + url + ", params: " + JSON.toJSONString(params)
+ ", custom headers: " + JSON.toJSONString(customHeaders)
+ ", conTimeout: " + conTimeout + ", readTimeout: "
+ readTimeout + ", encoding: " + requestEncoding + ".",
e);
}
}
}
throw new Exception("Excepion thrown at last after retried " + RETRY_TIMES + " times. Url: "
+ url + ", params: " + JSON.toJSONString(params) + ", custom headers: "
+ JSON.toJSONString(customHeaders) + ", conTimeout: " + conTimeout
+ ", readTimeout: " + readTimeout + ", encoding: " + requestEncoding
+ ".",
retryableEx);
}
/**
* 用于post上传文件，普通的post请求走另外一个接口
*
* @param url 请求的url
* @param httpEntity 请求的参数, 值可以是string或object[]
* @param customHeaders 自定义header, 值可以是string或string[]
* @param conTimeout socket连接超时时间, ms
* @param readTimeout 业务调用超时时间, ms
* @param requestEncoding 发起请求时使用的编码格式
* @return response
* @throws Exception
*/
public static CloseableHttpResponse post(String url, HttpEntity httpEntity,
Map<String, Object> customHeaders, int conTimeout,
int readTimeout,
String requestEncoding) throws Exception {
HttpPost post = new HttpPost(url);
post.setProtocolVersion(HttpVersion.HTTP_1_1);
post.setHeader(HTTP.CONN_DIRECTIVE, HTTP.CONN_KEEP_ALIVE);
// set header
setCustomHeaders(post, customHeaders);
post.setEntity(httpEntity);
// set config
post.setConfig(RequestConfig.custom().setConnectTimeout(conTimeout)
.setSocketTimeout(readTimeout).build());
// invoke
Exception retryableEx = null;
for (int i = 0; i < RETRY_TIMES; i++) {
try {
return httpClient.execute(post);
} catch (ClientProtocolException e) {
throw new Exception("Error posting files to url: " + url + ", custom headers: "
+ JSON.toJSONString(customHeaders) + ", conTimeout: "
+ conTimeout + ", readTimeout: " + readTimeout + ", encoding: "
+ requestEncoding + ".",
e);
} catch (IOException e) {
if (retryable(e)) {
retryableEx = e;
} else {
throw new Exception("Error posting files to url: " + url + ", params: "
+ ", custom headers: " + JSON.toJSONString(customHeaders)
+ ", conTimeout: " + conTimeout + ", readTimeout: "
+ readTimeout + ", encoding: " + requestEncoding + ".",
e);
}
}
}
throw new Exception("Excepion thrown at last after retried " + RETRY_TIMES + " times. Url: "
+ url + ", custom headers: " + JSON.toJSONString(customHeaders)
+ ", conTimeout: " + conTimeout + ", readTimeout: " + readTimeout
+ ", encoding: " + requestEncoding + ".",
retryableEx);
}
/**
* 发起get请求, 对连接类错误会自动重试，最多三次
*
* @param url 请求的url
* @param params 请求的参数, 值可以是string或object[]
* @param customHeaders 自定义header, 值可以是string或string[]
* @param conTimeout socket连接超时时间, ms
* @param readTimeout 业务调用超时时间, ms
* @param requestEncoding 发起请求时使用的编码格式
* @return response
* @throws Exception
*/
public static CloseableHttpResponse get(String url, Map<String, ?> params,
Map<String, Object> customHeaders, int conTimeout,
int readTimeout,
String requestEncoding) throws Exception {
// build query url
try {
URIBuilder builder = new URIBuilder(url);
builder.addParameters(buildParamList(params));
builder.setCharset(Charset.forName(requestEncoding));
url = builder.toString();
} catch (Exception e) {
throw e;
}
// logger.info("request url: "+url);
HttpGet get = new HttpGet(url);
get.setProtocolVersion(HttpVersion.HTTP_1_1);
get.setHeader(HTTP.CONN_DIRECTIVE, HTTP.CONN_KEEP_ALIVE);
// set header
setCustomHeaders(get, customHeaders);
// set config
get.setConfig(RequestConfig.custom().setConnectTimeout(conTimeout)
.setSocketTimeout(readTimeout).build());
// invoke
Exception retryableEx = null;
for (int i = 0; i < RETRY_TIMES; i++) {
try {
return httpClient.execute(get);
} catch (ClientProtocolException e) {
throw new Exception("Error requesting url: " + url + ", custom headers: "
+ JSON.toJSONString(customHeaders) + ", conTimeout: "
+ conTimeout + ", readTimeout: " + readTimeout + ", encoding: "
+ requestEncoding + ".",
e);
} catch (IOException e) {
if (retryable(e)) {
retryableEx = e;
} else {
throw new Exception("Error requesting url: " + url + ", custom headers: "
+ JSON.toJSONString(customHeaders) + ", conTimeout: "
+ conTimeout + ", readTimeout: " + readTimeout
+ ", encoding: " + requestEncoding + ".",
e);
}
}
}
throw new Exception("Excepion thrown at last after retried " + RETRY_TIMES + " times. Url: "
+ url + ", custom headers: " + JSON.toJSONString(customHeaders)
+ ", conTimeout: " + conTimeout + ", readTimeout: " + readTimeout
+ ", encoding: " + requestEncoding + ".",
retryableEx);
}
private static List<NameValuePair> buildParamList(Map<String, ?> params) {
List<NameValuePair> ret = new ArrayList<NameValuePair>();
if (params == null || params.isEmpty()) {
return ret;
}
for (Map.Entry<String, ?> e : params.entrySet()) {
Object v = e.getValue();
if (v instanceof Object[]) {
for (Object o : ((Object[]) v)) {
ret.add(new BasicNameValuePair(e.getKey(), String.valueOf(o)));
}
} else {
ret.add(new BasicNameValuePair(e.getKey(), String.valueOf(v)));
}
}
return ret;
}
private static void setCustomHeaders(HttpMessage msg, Map<String, Object> customHeaders) {
if (customHeaders == null || customHeaders.isEmpty()) {
return;
}
for (Map.Entry<String, Object> p : customHeaders.entrySet()) {
Object v = p.getValue();
if (v instanceof String[]) {
msg.addHeader(p.getKey(), StringUtils.join(Arrays.asList((String[]) v), ','));
} else {
msg.addHeader(p.getKey(), String.valueOf(v));
}
}
}
private static boolean retryable(IOException e) {
return e instanceof NoHttpResponseException || e instanceof ConnectTimeoutException
|| e instanceof UnknownHostException
|| e.getMessage() != null
&& e.getMessage().contains("java.net.UnknownHostException");
}
public static Boolean getHttpsHostnameVerifyIgnore() {
return HTTPS_HOSTNAME_VERIFY_IGNORE;
}
/**
* Setter method for property httpsHostnameVerifyIgnore.
*
* @param httpsHostnameVerifyIgnore value to be assigned to property httpsHostnameVerifyIgnore
*/
public static void setHttpsHostnameVerifyIgnore(Boolean httpsHostnameVerifyIgnore) {
HTTPS_HOSTNAME_VERIFY_IGNORE = httpsHostnameVerifyIgnore;
}
}
Raw
HttpInvokerTest.java
package test;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.http.HttpResponse;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.handler.AbstractHandler;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
/**
* @author pf-miles Sep 9, 2016 10:13:26 AM
*/
public class HttpInvokerTest {
private static Server server;
@BeforeClass
public static void init() throws Exception {
server = new Server(12345);
server.setHandler(new AbstractHandler() {
@Override
public void handle(String target, Request baseRequest, HttpServletRequest request,
HttpServletResponse response) throws IOException, ServletException {
StringBuilder sb = new StringBuilder();
sb.append("Headers:").append('\n');
Enumeration<String> hns = request.getHeaderNames();
while (hns.hasMoreElements()) {
String n = hns.nextElement();
String v = request.getHeader(n);
sb.append(n).append(": ").append(v).append('\n');
}
sb.append("Entity:\n");
String contentKvs = resolveCttKvs(request.getParameterMap());
sb.append(contentKvs).append('\n');
response.getWriter().write(sb.toString());
response.flushBuffer();
}
});
server.start();
}
@AfterClass
public static void destroy() throws Exception {
server.stop();
}
private static String resolveCttKvs(Map<String, String[]> paramMap) {
if (paramMap == null || paramMap.isEmpty())
return "";
StringBuilder sb = new StringBuilder();
for (Map.Entry<String, String[]> e : paramMap.entrySet()) {
sb.append(e.getKey()).append("=");
for (String v : e.getValue()) {
sb.append(v).append(';');
}
sb.append('\n');
}
return sb.toString();
}
@Test
public void testPost() throws Exception {
Map<String, Object> params = new HashMap<String, Object>();
params.put("single", "val");
params.put("multi", new Object[] { "multi", "vals" });
Map<String, Object> customHeaders = new HashMap<String, Object>();
customHeaders.put("single", "singleHeader");
customHeaders.put("multi", new String[] { "multi", "headers" });
HttpResponse resp = HttpInvoker.post("http://127.0.0.1:12345", params, customHeaders,
300000, 300000);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
resp.getEntity().writeTo(bos);
System.out.println(new String(bos.toByteArray(), SdkConfig.ENCODING));
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment