3

I am trying to write an Android app that can take a given video and change it's resolution to a given size, bit rate and audio sample rate. I am using the built-in MediaCodec and MediaMuxer classes provided in API level 18, and I am pretty much following the samples from BigFlake.com (http://bigflake.com/mediacodec/), but I am having some trouble getting it all to work smoothly.

Right now I am getting an IllegalStateException when trying to call the dequeueInputBuffer on the MediaCodec class. I know this is kind-of a catch all exception, but was hoping someone could take a look at my code below and let me know where I'm going wrong?

UPDATE

Turns out the issue with the dequeueInputBuffer call was the resolution. Since 480 x 360 isn't a multiple of 16 the dequeueInputBuffer threw the IllegalStateException. Changing my target resolution to 512 x 288 fixed the issue.

Now,I am having an issue with the queueInputBuffer method call. This call is giving me the exact same IllegalStateException That I was getting before, but now for different reasons.

Funny thing is I have looked at the examples on BigFlake.com, and even re-implemented it, and I still get the same exception on this line. Does anyone have any idea what's going on?

BTW, I have removed my old code and updated this post with my latest.

Thanks!

package com.mikesappshop.videoconverter; import android.media.MediaCodec; import android.media.MediaCodecInfo; import android.media.MediaCodecList; import android.media.MediaExtractor; import android.media.MediaFormat; import android.media.MediaMuxer; import android.util.Log; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; /** * Created by mneill on 11/3/15. */ public class VideoConverter { // Interface public interface CompletionHandler { void videoEncodingDidComplete(Error error); } // Constants private static final String TAG = "VideoConverter"; private static final boolean VERBOSE = true; // lots of logging // parameters for the encoder private static final String MIME_TYPE = "video/avc"; // H.264 Advanced Video Coding private static final int FRAME_RATE = 15; // 15fps private static final int CAPTURE_RATE = 15; // 15fps private static final int IFRAME_INTERVAL = 10; // 10 seconds between I-frames private static final int CHANNEL_COUNT = 1; private static final int SAMPLE_RATE = 128000; private static final int TIMEOUT_USEC = 10000; // size of a frame, in pixels private int mWidth = -1; private int mHeight = -1; // bit rate, in bits per second private int mBitRate = -1; // encoder / muxer state private MediaCodec mDecoder; private MediaCodec mEncoder; private MediaMuxer mMuxer; private int mTrackIndex; private boolean mMuxerStarted; /** * Starts encoding process */ public void convertVideo(String mediaFilePath, String destinationFilePath, CompletionHandler handler) { // TODO: Make configurable // mWidth = 480; // mHeight = 360; mWidth = 512; mHeight = 288; mBitRate = 500000; try { if ((mWidth % 16) != 0 || (mHeight % 16) != 0) { Log.e(TAG, "Width or Height not multiple of 16"); Error e = new Error("Width and height must be a multiple of 16"); handler.videoEncodingDidComplete(e); return; } // prep the decoder and the encoder prepareEncoderDecoder(destinationFilePath); // load file File file = new File(mediaFilePath); byte[] fileData = readContentIntoByteArray(file); // fill up the input buffer fillInputBuffer(fileData); // encode buffer encode(); } catch (Exception ex) { Log.e(TAG, ex.toString()); ex.printStackTrace(); } finally { // release encoder and muxer releaseEncoder(); } } /** * Configures encoder and muxer state */ private void prepareEncoderDecoder(String outputPath) throws Exception { // create decoder to read in the file data mDecoder = MediaCodec.createDecoderByType(MIME_TYPE); // create encoder to encode the file data into a new format MediaCodecInfo info = selectCodec(MIME_TYPE); int colorFormat = selectColorFormat(info, MIME_TYPE); MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight); format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat); format.setInteger(MediaFormat.KEY_BIT_RATE, mBitRate); format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE); format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL); mEncoder = MediaCodec.createByCodecName(info.getName()); mEncoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); mEncoder.start(); // Create a MediaMuxer for saving the data mMuxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); mTrackIndex = -1; mMuxerStarted = false; } /** * Releases encoder resources. May be called after partial / failed initialization. */ private void releaseEncoder() { if (VERBOSE) Log.d(TAG, "releasing encoder objects"); if (mEncoder != null) { mEncoder.stop(); mEncoder.release(); mEncoder = null; } if (mMuxer != null) { mMuxer.stop(); mMuxer.release(); mMuxer = null; } } private void fillInputBuffer(byte[] data) { boolean inputDone = false; int processedDataSize = 0; int frameIndex = 0; Log.d(TAG, "[fillInputBuffer] Buffer load start"); ByteBuffer[] inputBuffers = mEncoder.getInputBuffers(); while (!inputDone) { int inputBufferIndex = mEncoder.dequeueInputBuffer(10000); if (inputBufferIndex >= 0) { if (processedDataSize >= data.length) { mEncoder.queueInputBuffer(inputBufferIndex, 0, 0, computePresentationTime(frameIndex), MediaCodec.BUFFER_FLAG_END_OF_STREAM); inputDone = true; Log.d(TAG, "[fillInputBuffer] Buffer load complete"); } else { ByteBuffer inputBuffer = inputBuffers[inputBufferIndex]; int limit = inputBuffer.capacity(); int pos = frameIndex * limit; byte[] subData = new byte[limit]; System.arraycopy(data, pos, subData, 0, limit); inputBuffer.clear(); inputBuffer.put(subData); Log.d(TAG, "[encode] call queueInputBuffer"); mDecoder.queueInputBuffer(inputBufferIndex, 0, subData.length, computePresentationTime(frameIndex), MediaCodec.BUFFER_FLAG_CODEC_CONFIG); Log.d(TAG, "[encode] did call queueInputBuffer"); Log.d(TAG, "[encode] Loaded frame " + frameIndex + " into buffer"); frameIndex++; } } } } private void encode() throws Exception { // get buffer info MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); // start encoding ByteBuffer[] encoderOutputBuffers = mEncoder.getOutputBuffers(); while (true) { int encoderStatus = mEncoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC); if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) { // no output available yet if (VERBOSE) Log.d(TAG, "no output available, spinning to await EOS"); break; } else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { // not expected for an encoder encoderOutputBuffers = mEncoder.getOutputBuffers(); } else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { // should happen before receiving buffers, and should only happen once if (!mMuxerStarted) { MediaFormat newFormat = mEncoder.getOutputFormat(); Log.d(TAG, "encoder output format changed: " + newFormat); // now that we have the Magic Goodies, start the muxer mTrackIndex = mMuxer.addTrack(newFormat); mMuxer.start(); mMuxerStarted = true; } } else if (encoderStatus > 0) { ByteBuffer encodedData = encoderOutputBuffers[encoderStatus]; if (encodedData == null) { throw new RuntimeException("encoderOutputBuffer " + encoderStatus + " was null"); } if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { if (VERBOSE) Log.d(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG"); bufferInfo.size = 0; } if (bufferInfo.size != 0) { if (!mMuxerStarted) { throw new RuntimeException("muxer hasn't started"); } // adjust the ByteBuffer values to match BufferInfo (not needed?) encodedData.position(bufferInfo.offset); encodedData.limit(bufferInfo.offset + bufferInfo.size); mMuxer.writeSampleData(mTrackIndex, encodedData, bufferInfo); if (VERBOSE) Log.d(TAG, "sent " + bufferInfo.size + " bytes to muxer"); } mEncoder.releaseOutputBuffer(encoderStatus, false); if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { if (VERBOSE) Log.d(TAG, "end of stream reached"); break; // out of while } } } } private byte[] readContentIntoByteArray(File file) throws Exception { FileInputStream fileInputStream = null; byte[] bFile = new byte[(int) file.length()]; //convert file into array of bytes fileInputStream = new FileInputStream(file); fileInputStream.read(bFile); fileInputStream.close(); return bFile; } /** * Returns the first codec capable of encoding the specified MIME type, or null if no * match was found. */ private static MediaCodecInfo selectCodec(String mimeType) { int numCodecs = MediaCodecList.getCodecCount(); for (int i = 0; i < numCodecs; i++) { MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i); if (!codecInfo.isEncoder()) { continue; } String[] types = codecInfo.getSupportedTypes(); for (int j = 0; j < types.length; j++) { if (types[j].equalsIgnoreCase(mimeType)) { return codecInfo; } } } return null; } private static int selectColorFormat(MediaCodecInfo codecInfo, String mimeType) { MediaCodecInfo.CodecCapabilities capabilities = codecInfo.getCapabilitiesForType(mimeType); for (int i = 0; i < capabilities.colorFormats.length; i++) { int colorFormat = capabilities.colorFormats[i]; if (isRecognizedFormat(colorFormat)) { return colorFormat; } } return 0; // not reached } private static boolean isRecognizedFormat(int colorFormat) { switch (colorFormat) { // these are the formats we know how to handle for this test case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar: case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedPlanar: case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar: case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar: case MediaCodecInfo.CodecCapabilities.COLOR_TI_FormatYUV420PackedSemiPlanar: return true; default: return false; } } /** * Generates the presentation time for frame N, in microseconds. */ private static long computePresentationTime(int frameIndex) { return 132 + frameIndex * 1000000 / FRAME_RATE; } }
Improve this question
0

3 Answers 3

Reset to default
2

The only thing that leaps out at me is you're doing this:

MediaCodecInfo info = selectCodec(MIME_TYPE); int colorFormat = selectColorFormat(info, MIME_TYPE);

and then this:

mEncoder = MediaCodec.createEncoderByType(MIME_TYPE);

instead of this:

mEncoder = MediaCodec.createByCodecName(info.getName());

(see this example)

I don't know why that would lead to failure, or why the failure wouldn't manifest until you went to decode an input buffer. But if you're picking a color format from a codec, you should be sure to use that codec.

Other notes:

  • You're logging exceptions from mEncoder.configure() but not halting execution. Stick a throw new RuntimeException(ex) in there.
  • You're logging but ignoring exceptions in encode()... why? Drop the try/catch to ensure that failures are obvious and halt the encoding process.
  • You can omit KEY_SAMPLE_RATE and KEY_CHANNEL_COUNT, those are for audio.

Video conversions in which you also decode the video with MediaCodec are best structured with data passed through Surface, but it looks like you've already got the YUV data in a file. Hopefully you've selected a YUV format that matches what the encoder accepts.

Improve this answer
Sign up to request clarification or add additional context in comments.

3 Comments

Thanks for the answer @fadden, but unfortunately this wasn't the solution to my problem. I updated my post with the fix, but I am giving you an up vote because this is great information. I fixed a lot of the issues you pointed out, but I am still having issues :(
What device are you using? What version of Android? What is the codec name (from info.getName())? What color format is used? How much data are you putting into the buffer? Is there anything odd in logcat between the configure() and the line that throws the exception?
It's ok. I punted on the idea all together as I realized some of the APIs require level 23, and I need to support down to 16 so I can't use this implementation anyways. I ended up going with an FFMPEG integration using this framework github.com/WritingMinds/ffmpeg-android-java
0

In the function fillInputBuffer -> in the else part why are you enqueuing in the decoder buffer.

Log.d(TAG, "[encode] call queueInputBuffer"); mDecoder.queueInputBuffer(inputBufferIndex, 0, subData.length, computePresentationTime(frameIndex), MediaCodec.BUFFER_FLAG_CODEC_CONFIG); Log.d(TAG, "[encode] did call queueInputBuffer");

instead of mDecoder use mEncoder it will work

Improve this answer

Comments

0

You can change video resolution with Transformer API:

implementation("androidx.media3:media3-transformer:1.4.1") implementation("androidx.media3:media3-effect:1.4.1") implementation("androidx.media3:media3-common:1.4.1")

fun changeRes(context: Context, input: Uri) { val effect = arrayListOf<Effect>() effect.add( Presentation.createForWidthAndHeight( 1080, 1920, Presentation.LAYOUT_SCALE_TO_FIT ) ) val transformer = with( Transformer.Builder(context)) { addListener(object : Transformer.Listener { override fun onCompleted( composition: Composition, exportResult: ExportResult ) { Log.d(TAG, "onCompleted") } override fun onError( composition: Composition, exportResult: ExportResult, exportException: ExportException ) { Log.d(TAG, "onError") } }) setVideoMimeType(MimeTypes.VIDEO_H264) setEncoderFactory( DefaultEncoderFactory.Builder(context) .setRequestedVideoEncoderSettings( VideoEncoderSettings.Builder() .setBitrate(4 * 1024 * 1024) .build() ) .build() ) build() } val inputMediaItem = MediaItem.fromUri(input) val editedMediaItem = EditedMediaItem.Builder(inputMediaItem).apply { setEffects(Effects(mutableListOf(), effect)) } transformer.start(editedMediaItem.build(), outputFile.absolutePath) }

android docs: https://developer.android.com/media/media3/transformer/transformations https://android-developers.googleblog.com/2023/05/media-transcoding-and-editing-transform-and-roll-out.html

Improve this answer

1 Comment

While this link may answer the question, it is better to include the essential parts of the answer here and provide the link for reference. Link-only answers can become invalid if the linked page changes. - From Review

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.