How can I make a POST request in the same manner as Apache HttpClient lib, using HttpURLConnection

Question

I'm trying to make a POST request to a website. As the response to the POST request, I expect some JSON data.

Using Apache's HttpClient library, I am able to do this without any problems. The response data is JSON so I just parse it.

package com.mydomain.myapp; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; public class MyApp { private static String extract(String patternString, String target) { Pattern pattern = Pattern.compile(patternString); Matcher matcher = pattern.matcher(target); matcher.find(); return matcher.group(1); } private String getResponse(InputStream stream) throws Exception { BufferedReader in = new BufferedReader(new InputStreamReader(stream)); String inputLine; StringBuffer responseStringBuffer = new StringBuffer(); while ((inputLine = in.readLine()) != null) { responseStringBuffer.append(inputLine); } in.close(); return responseStringBuffer.toString(); } private final static String BASE_URL = "https://www.volkswagen-car-net.com"; private final static String BASE_GUEST_URL = "/portal/en_GB/web/guest/home"; private void run() throws Exception { CloseableHttpClient client = HttpClients.createDefault(); HttpGet httpGet = new HttpGet(BASE_URL + BASE_GUEST_URL); CloseableHttpResponse getResponse = client.execute(httpGet); HttpEntity responseEntity = getResponse.getEntity(); String data = getResponse(responseEntity.getContent()); EntityUtils.consume(responseEntity); String csrf = extract("<meta name=\"_csrf\" content=\"(.*)\"/>", data); System.out.println(csrf); HttpPost post = new HttpPost(BASE_URL + "/portal/web/guest/home/-/csrftokenhandling/get-login-url"); post.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"); post.setHeader("User-Agent'", "Mozilla/5.0 (Linux; Android 6.0.1; D5803 Build/23.5.A.1.291; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/63.0.3239.111 Mobile Safari/537.36"); post.setHeader("Referer", BASE_URL + "/portal"); post.setHeader("X-CSRF-Token", csrf); CloseableHttpResponse postResponse = client.execute(post); HttpEntity postResponseEntity = postResponse.getEntity(); String postData = getResponse(postResponseEntity.getContent()); System.out.println(postData); EntityUtils.consume(postResponseEntity); postResponse.close(); } public static void main(String[] args) throws Exception { MyApp myApp = new MyApp(); myApp.run(); } }

But I can't use the HttpClient library in my project. I need to be able to do the same thing with "just" HttpURLConnection.

But there is some magic going on with the HttpClient library that I cannot fathom. Because the response to my POST request using HttpURLConnection is just a redirect to a different webpage alltogheter.

Can someone point me in the right direction here?

Here's my current HttpURLConnection attempt:

package com.mydomain.myapp; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.URL; import java.util.regex.Matcher; import java.util.regex.Pattern; public class MyApp { private static String extract(String patternString, String target) { Pattern pattern = Pattern.compile(patternString); Matcher matcher = pattern.matcher(target); matcher.find(); return matcher.group(1); } private final static String BASE_URL = "https://www.volkswagen-car-net.com"; private final static String BASE_GUEST_URL = "/portal/en_GB/web/guest/home"; private String getResponse(InputStream stream) throws Exception { BufferedReader in = new BufferedReader(new InputStreamReader(stream)); String inputLine; StringBuffer responseStringBuffer = new StringBuffer(); while ((inputLine = in.readLine()) != null) { responseStringBuffer.append(inputLine); } in.close(); return responseStringBuffer.toString(); } private String getResponse(HttpURLConnection connection) throws Exception { return getResponse(connection.getInputStream()); } private void run() throws Exception { HttpURLConnection getConnection1; URL url = new URL(BASE_URL + BASE_GUEST_URL); getConnection1 = (HttpURLConnection) url.openConnection(); getConnection1.setRequestMethod("GET"); if (getConnection1.getResponseCode() != HttpURLConnection.HTTP_OK) { throw new Exception("Request failed"); } String response = getResponse(getConnection1); getConnection1.disconnect(); String csrf = extract("<meta name=\"_csrf\" content=\"(.*)\"/>", response); System.out.println(csrf); HttpURLConnection postRequest; URL url2 = new URL(BASE_URL + "/portal/web/guest/home/-/csrftokenhandling/get-login-url"); postRequest = (HttpURLConnection) url2.openConnection(); postRequest.setDoOutput(true); postRequest.setRequestMethod("POST"); postRequest.setInstanceFollowRedirects(false); postRequest.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"); postRequest.setRequestProperty("User-Agent'", "Mozilla/5.0 (Linux; Android 6.0.1; D5803 Build/23.5.A.1.291; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/63.0.3239.111 Mobile Safari/537.36"); postRequest.setRequestProperty("Referer", BASE_URL + "/portal"); postRequest.setRequestProperty("X-CSRF-Token", csrf); postRequest.disconnect(); } public static void main(String[] args) throws Exception { MyApp myApp = new MyApp(); myApp.run(); } }

You just have to write request body yourself and write it to outputstream — Antoniossss
– Antoniossss, Commented Oct 31, 2018 at 20:12
Since you want to parse pages maybe you could use Jsoup for that - it can connect to web resources as well. — Antoniossss
– Antoniossss, Commented Oct 31, 2018 at 20:13
Sounds like you also may need to manually handle the 302 redirect in your code and then navigate to it yourself in code - possible apaches lib is handling that for you? — JGlass
– JGlass, Commented Oct 31, 2018 at 20:51
@JGlass I think you're onto something. But I'm lost on how to handle it with a HttpURLConnection. I tried using postRequest.setInstanceFollowRedirects(false); but then I just get HTML data as the response. In the example with HttpClient I get the JSON data directly. — birgersp
– birgersp, Commented Nov 1, 2018 at 8:29
Let me know if the answer doesnt work for you and I'll be happy to look into things further! — JGlass
– JGlass, Commented Nov 1, 2018 at 17:52

JGlass · Accepted Answer · 2018-11-01 17:52:03Z

Courtesy of a great programmer resource, e.g. MKYong (you know you've run into his site before ;-)) and I'll go over the gist of it in case the link ever goes down.

Gist:

The HttpURLConnection‘s follow redirect is just an indicator, in fact it won’t help you to do the “real” http redirection, you still need to handle it manually.
If a server is redirected from the original URL to another URL, the response code should be 301: Moved Permanently or 302: Temporary Redirect. And you can get the new redirected url by reading the “Location” header of the HTTP response header.

For example, access to the normal HTTP twitter website – http://www.twitter.com , it will auto redirect to the HTTPS twitter website – https://www.twitter.com.

Sample code

package com.mkyong.http; import java.io.BufferedReader; import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.URL; public class HttpRedirectExample { public static void main(String[] args) { try { String url = "http://www.twitter.com"; URL obj = new URL(url); HttpURLConnection conn = (HttpURLConnection) obj.openConnection(); conn.setReadTimeout(5000); conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8"); conn.addRequestProperty("User-Agent", "Mozilla"); conn.addRequestProperty("Referer", "google.com"); System.out.println("Request URL ... " + url); boolean redirect = false; // normally, 3xx is redirect int status = conn.getResponseCode(); if (status != HttpURLConnection.HTTP_OK) { if (status == HttpURLConnection.HTTP_MOVED_TEMP || status == HttpURLConnection.HTTP_MOVED_PERM || status == HttpURLConnection.HTTP_SEE_OTHER) redirect = true; } System.out.println("Response Code ... " + status); if (redirect) { // get redirect url from "location" header field String newUrl = conn.getHeaderField("Location"); // get the cookie if need, for login String cookies = conn.getHeaderField("Set-Cookie"); // open the new connnection again conn = (HttpURLConnection) new URL(newUrl).openConnection(); conn.setRequestProperty("Cookie", cookies); conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8"); conn.addRequestProperty("User-Agent", "Mozilla"); conn.addRequestProperty("Referer", "google.com"); System.out.println("Redirect to URL : " + newUrl); } BufferedReader in = new BufferedReader( new InputStreamReader(conn.getInputStream())); String inputLine; StringBuffer html = new StringBuffer(); while ((inputLine = in.readLine()) != null) { html.append(inputLine); } in.close(); System.out.println("URL Content... \n" + html.toString()); System.out.println("Done"); } catch (Exception e) { e.printStackTrace(); } } }

Thanks @JGlass, but for some reason this does not yield the same result as when using HttpComponents. I'm completely stuck at this. The worst part is I really want to use HttpComponents, but it doesn't seem to play nice with Android...
So is the example code not redirecting for you? Are the URL's you have in your code valid URL's that I could try and get the redirect working for you, e.g. can I use them to test with?
The example code redirects me, but the response (after following the redirect) is not the same as when using HttpClient. The URLs are valid for testing.

Collectives™ on Stack Overflow

How can I make a POST request in the same manner as Apache HttpClient lib, using HttpURLConnection

1 Answer 1

3 Comments

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

3 Comments

Related